big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10038. [UI] Finish Time is not correctly parsed in the RM Apps page. Contributed by Inigo Goiri.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2019-12-19 11:37:17 -08:00
parent 52d7b745c6
commit ef59ffd362
4 changed files with 135 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
View File

@ -33,8 +33,10 @@ public static String appsTableInit(boolean isResourceManager) {
public static String appsTableInit(
boolean isFairSchedulerPage, boolean isResourceManager) {
// id, user, name, queue, starttime, finishtime, state, status, progress, ui
// id, user, name, app type, app tags, queue, priority,
// starttime, launchtime, finishtime, state, status, progress, ui
// FairSchedulerPage's table is a bit different
// This is define in RMAppsBlock.COLUMNS for the RM
return tableInit()
.append(", 'aaData': appsTableData")
.append(", bDeferRender: true")
@ -51,24 +53,26 @@ private static String getAppsTableColumnDefs(
String progressIndex = "[11]";
StringBuilder sb = new StringBuilder();
sb.append("[\n")
.append("{'sType':'natural', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'num-ignore-str', 'aTargets': [6, 7, 8]")
.append(", 'mRender': renderHadoopDate }");
.append("{'sType':'natural', 'aTargets': [0], ")
.append("'mRender': parseHadoopID },\n")
.append("{'sType':'num-ignore-str', 'aTargets': [7, 8, 9], ")
.append("'mRender': renderHadoopDate },\n");
if (isResourceManager) {
// Update following line if any column added in RM page before column 11
sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }");
// set progress column index to 18
progressIndex = "[18]";
sb.append("{'sType':'num-ignore-str', ")
.append("'aTargets': [12, 13, 14, 15, 16] },\n");
// set progress column index to 19
progressIndex = "[19]";
} else if (isFairSchedulerPage) {
// Update following line if any column added in scheduler page before column 11
sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }");
sb.append("{'sType':'num-ignore-str', ")
.append("'aTargets': [11, 12, 13, 14, 15] },\n");
// set progress column index to 16
progressIndex = "[16]";
}
sb.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
sb.append(progressIndex);
sb.append(", 'mRender': parseHadoopProgress }]");
sb.append("{'sType':'numeric', bSearchable:false, 'aTargets':")
.append(progressIndex)
.append(", 'mRender': parseHadoopProgress }\n]");
return sb.toString();
}

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ColumnHeader.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
/**
* Header for a Web UI column. This used with TH.
*/
public class ColumnHeader {
private String selector;
private String cdata;
public ColumnHeader(String pselector, String pcdata) {
this.selector = pselector;
this.cdata = pcdata;
}
/**
* Get the selector field for the TH.
* @return Selector.
*/
public String getSelector() {
return this.selector;
}
/**
* Get the cdata field for the TH.
* @return CData.
*/
public String getCData() {
return this.cdata;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
View File

@ -42,6 +42,8 @@
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.THEAD;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
import com.google.inject.Inject;
@ -49,6 +51,32 @@ public class RMAppsBlock extends AppsBlock {
private ResourceManager rm;
/** Columns for the Apps RM page. */
static final ColumnHeader[] COLUMNS = {
new ColumnHeader(".id", "ID"),
new ColumnHeader(".user", "User"),
new ColumnHeader(".name", "Name"),
new ColumnHeader(".type", "Application Type"),
new ColumnHeader(".apptag", "Application Tags"),
new ColumnHeader(".queue", "Queue"),
new ColumnHeader(".priority", "Application Priority"),
new ColumnHeader(".starttime", "StartTime"),
new ColumnHeader(".IDlaunchtime", "LaunchTime"),
new ColumnHeader(".finishtime", "FinishTime"),
new ColumnHeader(".state", "State"),
new ColumnHeader(".finalstatus", "FinalStatus"),
new ColumnHeader(".runningcontainer", "Running Containers"),
new ColumnHeader(".allocatedCpu", "Allocated CPU VCores"),
new ColumnHeader(".allocatedMemory", "Allocated Memory MB"),
new ColumnHeader(".reservedCpu", "Reserved CPU VCores"),
new ColumnHeader(".reservedMemory", "Reserved Memory MB"),
new ColumnHeader(".queuePercentage", "% of Queue"),
new ColumnHeader(".clusterPercentage", "% of Cluster"),
new ColumnHeader(".progress", "Progress"),
new ColumnHeader(".ui", "Tracking UI"),
new ColumnHeader(".blacklisted", "Blacklisted Nodes"),
};
@Inject
RMAppsBlock(ResourceManager rm, View.ViewContext ctx) {
super(null, ctx);
@ -57,26 +85,12 @@ public class RMAppsBlock extends AppsBlock {
@Override
protected void renderData(Block html) {
TBODY<TABLE<Hamlet>> tbody =
html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
.th(".name", "Name").th(".type", "Application Type")
.th(".apptag", "Application Tags")
.th(".queue", "Queue").th(".priority", "Application Priority")
.th(".starttime", "StartTime")
.th("launchtime", "LaunchTime")
.th(".finishtime", "FinishTime").th(".state", "State")
.th(".finalstatus", "FinalStatus")
.th(".runningcontainer", "Running Containers")
.th(".allocatedCpu", "Allocated CPU VCores")
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
.th(".ui", "Tracking UI")
.th(".blacklisted", "Blacklisted Nodes").__()
.__().tbody();
TR<THEAD<TABLE<Hamlet>>> tr = html.table("#apps").thead().tr();
for (ColumnHeader col : COLUMNS) {
tr = tr.th(col.getSelector(), col.getCData());
}
TBODY<TABLE<Hamlet>> tbody = tr.__().__().tbody();
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -21,12 +21,14 @@
import static org.apache.hadoop.yarn.server.resourcemanager.MockNodes.newResource;
import static org.apache.hadoop.yarn.webapp.Params.TITLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
@ -62,6 +64,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@ -159,6 +162,40 @@ public void configure(Binder binder) {
}
@Test
public void testRMAppColumnIndices() {
// Find the columns to check
List<Integer> colsId = new LinkedList<Integer>();
List<Integer> colsTime = new LinkedList<Integer>();
List<Integer> colsProgress = new LinkedList<Integer>();
for (int i = 0; i < RMAppsBlock.COLUMNS.length; i++) {
ColumnHeader col = RMAppsBlock.COLUMNS[i];
if (col.getCData().contains("ID")) {
colsId.add(i);
} else if (col.getCData().contains("Time")) {
colsTime.add(i);
} else if (col.getCData().contains("Progress")) {
colsProgress.add(i);
}
}
// Verify that the table JS header matches the columns
String tableInit = WebPageUtils.appsTableInit(true);
for (String tableLine : tableInit.split("\\n")) {
if (tableLine.contains("parseHadoopID")) {
assertTrue(tableLine + " should have id " + colsId,
tableLine.contains(colsId.toString()));
} else if (tableLine.contains("renderHadoopDate")) {
assertTrue(tableLine + " should have dates " + colsTime,
tableLine.contains(colsTime.toString()));
} else if (tableLine.contains("parseHadoopProgress")) {
assertTrue(tableLine + " should have progress " + colsProgress,
tableLine.contains(colsProgress.toString()));
}
}
}
public static RMContext mockRMContext(int numApps, int racks, int numNodes,
int mbsPerNode) {
final List<RMApp> apps = MockAsm.newApplications(numApps);