MAPREDUCE-5052. Job History UI and web services confusing job start time and job submit time (Chen He via jeagles)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1549896 13f79535-47bb-0310-9956-ffa450edef68
2013-12-10 17:17:47 +00:00 · 2013-12-10 17:17:47 +00:00 · ee5351bf22
commit ee5351bf22
parent a0c911a7f9
13 changed files with 66 additions and 8 deletions
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@ -178,6 +178,9 @@ Release 2.4.0 - UNRELEASED

    MAPREDUCE-5481. Enable uber jobs to have multiple reducers (Sandy Ryza)

+    MAPREDUCE-5052. Job History UI and web services confusing job start time and
+    job submit time (Chen He via jeagles)
+
  OPTIMIZATIONS

    MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@ -520,6 +520,11 @@ protected void handleEvent(JobHistoryEvent event) {
        mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime());
        mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName());
      }
+      //initialize the launchTime in the JobIndexInfo of MetaInfo
+      if(event.getHistoryEvent().getEventType() == EventType.JOB_INITED ){
+        JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent();
+        mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime());
+      }

      // If this is JobFinishedEvent, close the writer and setup the job-index
      if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
@ -155,6 +155,7 @@ public static JobId newJobID(ApplicationId appID, int i) {
  public static JobReport newJobReport(JobId id) {
    JobReport report = Records.newRecord(JobReport.class);
    report.setJobId(id);
+    report.setSubmitTime(System.currentTimeMillis()-DT);
    report
        .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
    report.setFinishTime(System.currentTimeMillis()
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
@ -51,6 +51,7 @@ public class FileNameIndexUtils {
  private static final int NUM_REDUCES_INDEX = 6;
  private static final int JOB_STATUS_INDEX = 7;
  private static final int QUEUE_NAME_INDEX = 8;
+  private static final int JOB_START_TIME_INDEX = 9;

  /**
   * Constructs the job history file name from the JobIndexInfo.
@ -64,7 +65,7 @@ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException
    sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
    sb.append(DELIMITER);
    
-    //StartTime
+    //SubmitTime
    sb.append(indexInfo.getSubmitTime());
    sb.append(DELIMITER);
    
@ -94,6 +95,10 @@ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException
    
    //QueueName
    sb.append(indexInfo.getQueueName());
+    sb.append(DELIMITER);
+
+    //JobStartTime
+    sb.append(indexInfo.getJobStartTime());

    sb.append(JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION);
    return encodeJobHistoryFileName(sb.toString());
@ -161,6 +166,14 @@ public static JobIndexInfo getIndexInfo(String jhFileName) throws IOException {

      indexInfo.setQueueName(
          decodeJobHistoryFileName(jobDetails[QUEUE_NAME_INDEX]));
+
+      try{
+        indexInfo.setJobStartTime(
+          Long.parseLong(decodeJobHistoryFileName(jobDetails[JOB_START_TIME_INDEX])));
+      } catch (NumberFormatException e){
+        LOG.warn("Unable to parse launch time from job history file "
+            + jhFileName + " : " + e);
+      }
    } catch (IndexOutOfBoundsException e) {
      LOG.warn("Parsing job history file with partial data encoded into name: "
          + jhFileName);
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
@ -34,6 +34,7 @@ public class JobIndexInfo {
  private int numMaps;
  private int numReduces;
  private String jobStatus;
+  private long jobStartTime;
  
  public JobIndexInfo() {
  }
@ -48,6 +49,7 @@ public JobIndexInfo(long submitTime, long finishTime, String user,
    this.numMaps = numMaps;
    this.numReduces = numReduces;
    this.jobStatus = jobStatus;
+    this.jobStartTime = -1;
  }
  
  public long getSubmitTime() {
@ -104,6 +106,12 @@ public String getJobStatus() {
  public void setJobStatus(String jobStatus) {
    this.jobStatus = jobStatus;
  }
+  public long getJobStartTime() {
+      return jobStartTime;
+  }
+  public void setJobStartTime(long lTime) {
+      this.jobStartTime = lTime;
+  }

  @Override
  public String toString() {
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
@ -48,6 +48,7 @@ public class TestFileNameIndexUtils {
    + FileNameIndexUtils.DELIMITER + "%s"
    + FileNameIndexUtils.DELIMITER + "%s"
    + FileNameIndexUtils.DELIMITER + "%s"
+    + FileNameIndexUtils.DELIMITER + "%s"
    + JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION;

  private static final String JOB_ID = "job_1317928501754_0001";
@ -67,6 +68,7 @@ public class TestFileNameIndexUtils {
  private static final String NUM_REDUCES = "1";
  private static final String JOB_STATUS = "SUCCEEDED";
  private static final String QUEUE_NAME = "default";
+  private static final String JOB_START_TIME = "1317928742060";

  @Test
  public void testEncodingDecodingEquivalence() throws IOException {
@ -82,6 +84,7 @@ public void testEncodingDecodingEquivalence() throws IOException {
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
+    info.setJobStartTime(Long.parseLong(JOB_START_TIME));

    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
    JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
@ -104,6 +107,8 @@ public void testEncodingDecodingEquivalence() throws IOException {
        info.getJobStatus(), parsedInfo.getJobStatus());
    Assert.assertEquals("Queue name different after encoding and decoding",
        info.getQueueName(), parsedInfo.getQueueName());
+    Assert.assertEquals("Job start time different after encoding and decoding",
+              info.getJobStartTime(), parsedInfo.getJobStartTime());
  }

  @Test
@ -120,6 +125,7 @@ public void testUserNamePercentEncoding() throws IOException {
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
+    info.setJobStartTime(Long.parseLong(JOB_START_TIME));

    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
    Assert.assertTrue("User name not encoded correctly into job history file",
@ -137,7 +143,8 @@ public void testUserNamePercentDecoding() throws IOException {
        NUM_MAPS,
        NUM_REDUCES,
        JOB_STATUS,
-        QUEUE_NAME);
+        QUEUE_NAME,
+        JOB_START_TIME);

    JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
    Assert.assertEquals("User name doesn't match",
@ -158,6 +165,7 @@ public void testJobNamePercentEncoding() throws IOException {
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
+    info.setJobStartTime(Long.parseLong(JOB_START_TIME));

    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
    Assert.assertTrue("Job name not encoded correctly into job history file",
@ -175,7 +183,8 @@ public void testJobNamePercentDecoding() throws IOException {
        NUM_MAPS,
        NUM_REDUCES,
        JOB_STATUS,
-        QUEUE_NAME);
+        QUEUE_NAME,
+        JOB_START_TIME );

    JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
    Assert.assertEquals("Job name doesn't match",
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
@ -53,7 +53,8 @@ public PartialJob(JobIndexInfo jobIndexInfo, JobId jobId) {
    this.jobIndexInfo = jobIndexInfo;
    this.jobId = jobId;
    jobReport = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobReport.class);
-    jobReport.setStartTime(jobIndexInfo.getSubmitTime());
+    jobReport.setSubmitTime(jobIndexInfo.getSubmitTime());
+    jobReport.setStartTime(jobIndexInfo.getJobStartTime());
    jobReport.setFinishTime(jobIndexInfo.getFinishTime());
    jobReport.setJobState(getState());
  }
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
@ -84,6 +84,7 @@ public class HsJobBlock extends HtmlBlock {
        _("Queue:", job.getQueueName()).
        _("State:", job.getState()).
        _("Uberized:", job.isUber()).
+        _("Submitted:", new Date(job.getSubmitTime())).
        _("Started:", new Date(job.getStartTime())).
        _("Finished:", new Date(job.getFinishTime())).
        _("Elapsed:", StringUtils.formatTime(
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
@ -55,6 +55,7 @@ public class HsJobsBlock extends HtmlBlock {
      table("#jobs").
        thead().
          tr().
+            th("Submit Time").
            th("Start Time").
            th("Finish Time").
            th(".id", "Job ID").
@ -74,6 +75,7 @@ public class HsJobsBlock extends HtmlBlock {
    for (Job j : appContext.getAllJobs().values()) {
      JobInfo job = new JobInfo(j);
      jobsTableData.append("[\"")
+      .append(dateFormat.format(new Date(job.getSubmitTime()))).append("\",\"")
      .append(dateFormat.format(new Date(job.getStartTime()))).append("\",\"")
      .append(dateFormat.format(new Date(job.getFinishTime()))).append("\",\"")
      .append("<a href='").append(url("job", job.getId())).append("'>")
@ -101,6 +103,7 @@ public class HsJobsBlock extends HtmlBlock {
    tbody._().
    tfoot().
      tr().
+        th().input("search_init").$type(InputType.text).$name("submit_time").$value("Submit Time")._()._().
        th().input("search_init").$type(InputType.text).$name("start_time").$value("Start Time")._()._().
        th().input("search_init").$type(InputType.text).$name("finish_time").$value("Finish Time")._()._().
        th().input("search_init").$type(InputType.text).$name("start_time").$value("Job ID")._()._().
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
@ -44,6 +44,7 @@
@XmlAccessorType(XmlAccessType.FIELD)
 public class JobInfo {

+  protected long submitTime;
  protected long startTime;
  protected long finishTime;
  protected String id;
@ -85,6 +86,7 @@ public JobInfo(Job job) {
    this.mapsCompleted = job.getCompletedMaps();
    this.reducesTotal = job.getTotalReduces();
    this.reducesCompleted = job.getCompletedReduces();
+    this.submitTime = report.getSubmitTime();
    this.startTime = report.getStartTime();
    this.finishTime = report.getFinishTime();
    this.name = job.getName().toString();
@ -216,6 +218,10 @@ public String getId() {
    return this.id;
  }

+  public long getSubmitTime() {
+      return this.submitTime;
+  }
+
  public long getStartTime() {
    return this.startTime;
  }
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
@ -86,6 +86,7 @@ private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
          report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id, 
          mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
          String.valueOf(mockJob.getState()));
+      info.setJobStartTime(report.getStartTime());
      info.setQueueName(mockJob.getQueueName());
      ret.partial.put(id, new PartialJob(info, id));

--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java
@ -33,7 +33,7 @@
 public class VerifyJobsUtils {

  public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONException {
-    assertEquals("incorrect number of elements", 11, info.length());
+    assertEquals("incorrect number of elements", 12, info.length());

    // everyone access fields
    verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
@ -45,7 +45,7 @@ public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONExcep
  }
  
  public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
-    assertEquals("incorrect number of elements", 24, info.length());
+    assertEquals("incorrect number of elements", 25, info.length());

    // everyone access fields
    verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
@ -201,6 +201,7 @@ History Server REST API's.
   "jobs" : {
      "job" : [
         {
+            "submitTime" : 1326381344449,
            "state" : "SUCCEEDED",
            "user" : "user1",
            "reducesTotal" : 1,
@ -214,6 +215,7 @@ History Server REST API's.
            "finishTime" : 1326381356010
         },
         {
+            "submitTime" : 1326381446500
            "state" : "SUCCEEDED",
            "user" : "user1",
            "reducesTotal" : 1,
@ -255,6 +257,7 @@ History Server REST API's.
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <jobs>
  <job>
+    <submitTime>1326381344449</submitTime>
    <startTime>1326381344489</startTime>
    <finishTime>1326381356010</finishTime>
    <id>job_1326381300833_1_1</id>
@ -268,6 +271,7 @@ History Server REST API's.
    <reducesCompleted>1</reducesCompleted>
  </job>
  <job>
+    <submitTime>1326381446500</submitTime>
    <startTime>1326381446529</startTime>
    <finishTime>1326381582106</finishTime>
    <id>job_1326381300833_2_2</id>
@ -322,6 +326,8 @@ History Server REST API's.
 *---------------+--------------+-------------------------------+
 | diagnostics | string | A diagnostic message |
 *---------------+--------------+-------------------------------+
+| submitTime | long | The time the job submitted (in ms since epoch)|
+*---------------+--------------+-------------------------------+
 | startTime | long | The time the job started (in ms since epoch)|
 *---------------+--------------+-------------------------------+
 | finishTime | long | The time the job finished (in ms since epoch)|
@ -393,6 +399,7 @@ History Server REST API's.
 +---+
 {
   "job" : {
+      "submitTime":  1326381446500,
      "avgReduceTime" : 124961,
      "failedReduceAttempts" : 0,
      "state" : "SUCCEEDED",
@ -453,6 +460,7 @@ History Server REST API's.
 +---+
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <job>
+  <submitTime>1326381446500</submitTime>
  <startTime>1326381446529</startTime>
  <finishTime>1326381582106</finishTime>
  <id>job_1326381300833_2_2</id>
@ -2664,4 +2672,3 @@ History Server REST API's.
  </taskAttemptCounterGroup>
 </jobTaskAttemptCounters>
 +---+
-