From d2b9a604bd0e27509f1fa207e04b95312bd049c2 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 16 Nov 2012 17:30:22 +0000 Subject: [PATCH] MAPREDUCE-4802. Takes a long time to load the task list on the AM for large jobs (Ravi Prakash via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1410498 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/v2/app/webapp/TasksBlock.java | 53 +++++++++---------- .../mapreduce/v2/app/webapp/TasksPage.java | 27 +++++++--- .../mapreduce/v2/hs/webapp/HsJobsBlock.java | 6 +-- .../mapreduce/v2/hs/webapp/HsTasksBlock.java | 1 - .../webapps/static/yarn.dt.plugins.js | 6 ++- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a3a0c9fabf..e70db35f00 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -587,6 +587,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES + MAPREDUCE-4802. Takes a long time to load the task list on the AM for + large jobs (Ravi Prakash via bobby) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java index ccef4625eb..dc6bbb24d4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java @@ -21,15 +21,13 @@ import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.util.StringHelper.percent; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; -import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; @@ -66,6 +64,8 @@ public class TasksBlock extends HtmlBlock { th("Finish Time"). th("Elapsed Time")._()._(). tbody(); + StringBuilder tasksTableData = new StringBuilder("[\n"); + for (Task task : app.getJob().getTasks().values()) { if (type != null && task.getType() != type) { continue; @@ -73,31 +73,28 @@ public class TasksBlock extends HtmlBlock { TaskInfo info = new TaskInfo(task); String tid = info.getId(); String pct = percent(info.getProgress() / 100); - long startTime = info.getStartTime(); - long finishTime = info.getFinishTime(); - long elapsed = info.getElapsedTime(); - tbody. - tr(). - td(). - br().$title(String.valueOf(info.getTaskNum()))._(). // sorting - a(url("task", tid), tid)._(). - td(). - br().$title(pct)._(). - div(_PROGRESSBAR). - $title(join(pct, '%')). // tooltip - div(_PROGRESSBAR_VALUE). - $style(join("width:", pct, '%'))._()._()._(). - td(info.getState()). - td(). - br().$title(String.valueOf(startTime))._(). - _(Times.format(startTime))._(). - td(). - br().$title(String.valueOf(finishTime))._(). - _(Times.format(finishTime))._(). - td(). - br().$title(String.valueOf(elapsed))._(). - _(StringUtils.formatTime(elapsed))._()._(); + tasksTableData.append("[\"").append(tid).append("\",\"") + //Progress bar + .append("
").append("
\",\"") + + .append(info.getState()).append("\",\"") + .append(info.getStartTime()).append("\",\"") + .append(info.getFinishTime()).append("\",\"") + .append(info.getElapsedTime()).append("\"],\n"); } + //Remove the last comma and close off the array of arrays + if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') { + tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1); + } + tasksTableData.append("]"); + html.script().$type("text/javascript"). + _("var tasksTableData=" + tasksTableData)._(); + tbody._()._(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java index 3cdac512a3..3753b1ea39 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java @@ -37,11 +37,26 @@ public class TasksPage extends AppView { } private String tasksTableInit() { - return tableInit(). - // Sort by id upon page load - append(", aaSorting: [[0, 'asc']]"). - append(",aoColumns:[{sType:'title-numeric'},{sType:'title-numeric',"). - append("bSearchable:false},null,{sType:'title-numeric'},"). - append("{sType:'title-numeric'},{sType:'title-numeric'}]}").toString(); + return tableInit() + .append(", 'aaData': tasksTableData") + .append(", bDeferRender: true") + .append(", bProcessing: true") + + .append("\n, aoColumnDefs: [\n") + .append("{'sType':'numeric', 'aTargets': [0]") + .append(", 'mRender': parseHadoopID }") + + .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [1]") + .append(", 'mRender': parseHadoopProgress }") + + + .append("\n, {'sType':'numeric', 'aTargets': [3, 4]") + .append(", 'mRender': renderHadoopDate }") + + .append("\n, {'sType':'numeric', 'aTargets': [5]") + .append(", 'mRender': renderHadoopElapsedTime }]") + + // Sort by id upon page load + .append(", aaSorting: [[0, 'asc']] }").toString(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java index 53dd2ffd4c..95715c7b74 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java @@ -78,11 +78,11 @@ public class HsJobsBlock extends HtmlBlock { .append(dateFormat.format(new Date(job.getFinishTime()))).append("\",\"") .append("") .append(job.getId()).append("\",\"") - .append(StringEscapeUtils.escapeJavaScript(job.getName())) + .append(StringEscapeUtils.escapeHtml(job.getName())) .append("\",\"") - .append(StringEscapeUtils.escapeJavaScript(job.getUserName())) + .append(StringEscapeUtils.escapeHtml(job.getUserName())) .append("\",\"") - .append(StringEscapeUtils.escapeJavaScript(job.getQueueName())) + .append(StringEscapeUtils.escapeHtml(job.getQueueName())) .append("\",\"") .append(job.getState()).append("\",\"") .append(String.valueOf(job.getMapsTotal())).append("\",\"") diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java index d11dbffac0..a76e4bebf3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java @@ -29,7 +29,6 @@ import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js index f051bcd145..17c1b27d0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js @@ -58,10 +58,14 @@ jQuery.fn.dataTableExt.oApi.fnSetFilteringDelay = function ( oSettings, iDelay ) function renderHadoopDate(data, type, full) { if (type === 'display') { + if(data === '0') { + return "N/A"; + } return new Date(parseInt(data)).toUTCString(); } // 'filter', 'sort', 'type' and undefined all just use the number - return data; + // If date is 0, then for purposes of sorting it should be consider max_int + return data === '0' ? '9007199254740992' : data; } function renderHadoopElapsedTime(data, type, full) {