big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4802. Takes a long time to load the task list on the AM for large jobs (Ravi Prakash via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1410498 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-11-16 17:30:22 +00:00
parent f13da263bf
commit d2b9a604bd
6 changed files with 57 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -587,6 +587,9 @@ Release 0.23.6 - UNRELEASED
BUG FIXES
MAPREDUCE-4802. Takes a long time to load the task list on the AM for
large jobs (Ravi Prakash via bobby)
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES

53
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
View File

@ -21,15 +21,13 @@
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.percent;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@ -66,6 +64,8 @@ public class TasksBlock extends HtmlBlock {
th("Finish Time").
th("Elapsed Time")._()._().
tbody();
StringBuilder tasksTableData = new StringBuilder("[\n");
for (Task task : app.getJob().getTasks().values()) {
if (type != null && task.getType() != type) {
continue;
@ -73,31 +73,28 @@ public class TasksBlock extends HtmlBlock {
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
String pct = percent(info.getProgress() / 100);
long startTime = info.getStartTime();
long finishTime = info.getFinishTime();
long elapsed = info.getElapsedTime();
tbody.
tr().
td().
br().$title(String.valueOf(info.getTaskNum()))._(). // sorting
a(url("task", tid), tid)._().
td().
br().$title(pct)._().
div(_PROGRESSBAR).
$title(join(pct, '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", pct, '%'))._()._()._().
td(info.getState()).
td().
br().$title(String.valueOf(startTime))._().
_(Times.format(startTime))._().
td().
br().$title(String.valueOf(finishTime))._().
_(Times.format(finishTime))._().
td().
br().$title(String.valueOf(elapsed))._().
_(StringUtils.formatTime(elapsed))._()._();
tasksTableData.append("[\"<a href='").append(url("task", tid))
.append("'>").append(tid).append("</a>\",\"")
//Progress bar
.append("<br title='").append(pct)
.append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
.append(join(pct, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
.append(info.getState()).append("\",\"")
.append(info.getStartTime()).append("\",\"")
.append(info.getFinishTime()).append("\",\"")
.append(info.getElapsedTime()).append("\"],\n");
}
//Remove the last comma and close off the array of arrays
if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1);
}
tasksTableData.append("]");
html.script().$type("text/javascript").
_("var tasksTableData=" + tasksTableData)._();
tbody._()._();
}
}

27
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java
View File

@ -37,11 +37,26 @@ public class TasksPage extends AppView {
}
private String tasksTableInit() {
return tableInit().
// Sort by id upon page load
append(", aaSorting: [[0, 'asc']]").
append(",aoColumns:[{sType:'title-numeric'},{sType:'title-numeric',").
append("bSearchable:false},null,{sType:'title-numeric'},").
append("{sType:'title-numeric'},{sType:'title-numeric'}]}").toString();
return tableInit()
.append(", 'aaData': tasksTableData")
.append(", bDeferRender: true")
.append(", bProcessing: true")
.append("\n, aoColumnDefs: [\n")
.append("{'sType':'numeric', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [1]")
.append(", 'mRender': parseHadoopProgress }")
.append("\n, {'sType':'numeric', 'aTargets': [3, 4]")
.append(", 'mRender': renderHadoopDate }")
.append("\n, {'sType':'numeric', 'aTargets': [5]")
.append(", 'mRender': renderHadoopElapsedTime }]")
// Sort by id upon page load
.append(", aaSorting: [[0, 'asc']] }").toString();
}
}

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
View File

@ -78,11 +78,11 @@ public class HsJobsBlock extends HtmlBlock {
.append(dateFormat.format(new Date(job.getFinishTime()))).append("\",\"")
.append("<a href='").append(url("job", job.getId())).append("'>")
.append(job.getId()).append("</a>\",\"")
.append(StringEscapeUtils.escapeJavaScript(job.getName()))
.append(StringEscapeUtils.escapeHtml(job.getName()))
.append("\",\"")
.append(StringEscapeUtils.escapeJavaScript(job.getUserName()))
.append(StringEscapeUtils.escapeHtml(job.getUserName()))
.append("\",\"")
.append(StringEscapeUtils.escapeJavaScript(job.getQueueName()))
.append(StringEscapeUtils.escapeHtml(job.getQueueName()))
.append("\",\"")
.append(job.getState()).append("\",\"")
.append(String.valueOf(job.getMapsTotal())).append("\",\"")

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
View File

@ -29,7 +29,6 @@
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js
View File

@ -58,10 +58,14 @@ jQuery.fn.dataTableExt.oApi.fnSetFilteringDelay = function ( oSettings, iDelay )
function renderHadoopDate(data, type, full) {
if (type === 'display') {
if(data === '0') {
return "N/A";
}
return new Date(parseInt(data)).toUTCString();
}
// 'filter', 'sort', 'type' and undefined all just use the number
return data;
// If date is 0, then for purposes of sorting it should be consider max_int
return data === '0' ? '9007199254740992' : data;
}
function renderHadoopElapsedTime(data, type, full) {