diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 47b2064446..60d49e93eb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -176,6 +176,10 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-2855. Passing a cached class-loader to ResourceBundle creator to minimize counter names lookup time. (Siddarth Seth via vinodkv) + MAPREDUCE-3944. Change the history jobs/ webservice to return partial job + info for a significant performance improvement. (Robert Joseph Evans via + sseth) + BUG FIXES MAPREDUCE-3918 proc_historyserver no longer in command line arguments for HistoryServer (Jon Eagles via bobby) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index dcd2cf571f..11d19d5c52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -287,7 +287,7 @@ private void loadAllTasks() { } //History data is leisurely loaded when task level data is requested - private synchronized void loadFullHistoryData(boolean loadTasks, + protected synchronized void loadFullHistoryData(boolean loadTasks, Path historyFileAbsolute) throws IOException { LOG.info("Loading history file: [" + historyFileAbsolute + "]"); if (this.jobInfo != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 7f7bbcca82..944a28ee8d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; +import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; @@ -184,35 +185,32 @@ public JobsInfo getJobs(@QueryParam("user") String userQuery, break; } - // getAllJobs only gives you a partial we want a full - Job fullJob = appCtx.getJob(job.getID()); - if (fullJob == null) { - continue; - } - - JobInfo jobInfo = new JobInfo(fullJob); // can't really validate queue is a valid one since queues could change if (queueQuery != null && !queueQuery.isEmpty()) { - if (!jobInfo.getQueueName().equals(queueQuery)) { + if (!job.getQueueName().equals(queueQuery)) { continue; } } if (userQuery != null && !userQuery.isEmpty()) { - if (!jobInfo.getUserName().equals(userQuery)) { + if (!job.getUserName().equals(userQuery)) { continue; } } + JobReport report = job.getReport(); + if (checkStart - && (jobInfo.getStartTime() < sBegin || jobInfo.getStartTime() > sEnd)) { + && (report.getStartTime() < sBegin || report.getStartTime() > sEnd)) { continue; } if (checkEnd - && (jobInfo.getFinishTime() < fBegin || jobInfo.getFinishTime() > fEnd)) { + && (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd)) { continue; } - + + JobInfo jobInfo = new JobInfo(job); + allJobs.add(jobInfo); num++; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java index 03d97092e0..e5358dcb3e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java @@ -55,18 +55,18 @@ public class JobInfo { protected int mapsCompleted; protected int reducesTotal; protected int reducesCompleted; - protected boolean uberized; + protected Boolean uberized; protected String diagnostics; - protected long avgMapTime = 0; - protected long avgReduceTime = 0; - protected long avgShuffleTime = 0; - protected long avgMergeTime = 0; - protected int failedReduceAttempts = 0; - protected int killedReduceAttempts = 0; - protected int successfulReduceAttempts = 0; - protected int failedMapAttempts = 0; - protected int killedMapAttempts = 0; - protected int successfulMapAttempts = 0; + protected Long avgMapTime; + protected Long avgReduceTime; + protected Long avgShuffleTime; + protected Long avgMergeTime; + protected Integer failedReduceAttempts; + protected Integer killedReduceAttempts; + protected Integer successfulReduceAttempts; + protected Integer failedMapAttempts; + protected Integer killedMapAttempts; + protected Integer successfulMapAttempts; protected ArrayList acls; @XmlTransient @@ -80,7 +80,7 @@ public JobInfo() { public JobInfo(Job job) { this.id = MRApps.toString(job.getID()); JobReport report = job.getReport(); - countTasksAndAttempts(job); + this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.reducesTotal = job.getTotalReduces(); @@ -91,19 +91,33 @@ public JobInfo(Job job) { this.queue = job.getQueueName(); this.user = job.getUserName(); this.state = job.getState().toString(); - this.uberized = job.isUber(); - this.diagnostics = ""; - List diagnostics = job.getDiagnostics(); - if (diagnostics != null && !diagnostics.isEmpty()) { - StringBuffer b = new StringBuffer(); - for (String diag : diagnostics) { - b.append(diag); - } - this.diagnostics = b.toString(); - } this.acls = new ArrayList(); + if (job instanceof CompletedJob) { + avgMapTime = 0l; + avgReduceTime = 0l; + avgShuffleTime = 0l; + avgMergeTime = 0l; + failedReduceAttempts = 0; + killedReduceAttempts = 0; + successfulReduceAttempts = 0; + failedMapAttempts = 0; + killedMapAttempts = 0; + successfulMapAttempts = 0; + countTasksAndAttempts(job); + this.uberized = job.isUber(); + this.diagnostics = ""; + List diagnostics = job.getDiagnostics(); + if (diagnostics != null && !diagnostics.isEmpty()) { + StringBuffer b = new StringBuffer(); + for (String diag : diagnostics) { + b.append(diag); + } + this.diagnostics = b.toString(); + } + + Map allacls = job.getJobACLs(); if (allacls != null) { for (Map.Entry entry : allacls.entrySet()) { @@ -122,43 +136,43 @@ public long getNumReduces() { return numReduces; } - public long getAvgMapTime() { + public Long getAvgMapTime() { return avgMapTime; } - public long getAvgReduceTime() { + public Long getAvgReduceTime() { return avgReduceTime; } - public long getAvgShuffleTime() { + public Long getAvgShuffleTime() { return avgShuffleTime; } - public long getAvgMergeTime() { + public Long getAvgMergeTime() { return avgMergeTime; } - public long getFailedReduceAttempts() { + public Integer getFailedReduceAttempts() { return failedReduceAttempts; } - public long getKilledReduceAttempts() { + public Integer getKilledReduceAttempts() { return killedReduceAttempts; } - public long getSuccessfulReduceAttempts() { + public Integer getSuccessfulReduceAttempts() { return successfulReduceAttempts; } - public long getFailedMapAttempts() { + public Integer getFailedMapAttempts() { return failedMapAttempts; } - public long getKilledMapAttempts() { + public Integer getKilledMapAttempts() { return killedMapAttempts; } - public long getSuccessfulMapAttempts() { + public Integer getSuccessfulMapAttempts() { return successfulMapAttempts; } @@ -210,7 +224,7 @@ public long getFinishTime() { return this.finishTime; } - public boolean isUber() { + public Boolean isUber() { return this.uberized; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java new file mode 100644 index 0000000000..b8b64a6d91 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java @@ -0,0 +1,192 @@ +package org.apache.hadoop.mapreduce.v2.hs; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.JobACL; +import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.JobReport; +import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; +import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskType; +import org.apache.hadoop.mapreduce.v2.app.MockJobs; +import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.Task; +import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +import com.google.common.collect.Maps; + +public class MockHistoryJobs extends MockJobs { + + public static class JobsPair { + public Map partial; + public Map full; + } + + public static JobsPair newHistoryJobs(int numJobs, int numTasksPerJob, + int numAttemptsPerTask) throws IOException { + Map mocked = newJobs(numJobs, numTasksPerJob, numAttemptsPerTask); + return split(mocked); + } + + public static JobsPair newHistoryJobs(ApplicationId appID, int numJobsPerApp, + int numTasksPerJob, int numAttemptsPerTask) throws IOException { + Map mocked = newJobs(appID, numJobsPerApp, numTasksPerJob, + numAttemptsPerTask); + return split(mocked); + } + + private static JobsPair split(Map mocked) throws IOException { + JobsPair ret = new JobsPair(); + ret.full = Maps.newHashMap(); + ret.partial = Maps.newHashMap(); + for(Map.Entry entry: mocked.entrySet()) { + JobId id = entry.getKey(); + Job j = entry.getValue(); + ret.full.put(id, new MockCompletedJob(j)); + JobReport report = j.getReport(); + JobIndexInfo info = new JobIndexInfo(report.getStartTime(), + report.getFinishTime(), j.getUserName(), j.getName(), id, + j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState())); + info.setQueueName(j.getQueueName()); + ret.partial.put(id, new PartialJob(info, id)); + } + return ret; + } + + private static class MockCompletedJob extends CompletedJob { + private Job job; + + public MockCompletedJob(Job job) throws IOException { + super(new Configuration(), job.getID(), null, true, job.getUserName(), + null, null); + this.job = job; + } + + @Override + public int getCompletedMaps() { + return job.getCompletedMaps(); + } + + @Override + public int getCompletedReduces() { + return job.getCompletedReduces(); + } + + @Override + public org.apache.hadoop.mapreduce.Counters getAllCounters() { + return job.getAllCounters(); + } + + @Override + public JobId getID() { + return job.getID(); + } + + @Override + public JobReport getReport() { + return job.getReport(); + } + + @Override + public float getProgress() { + return job.getProgress(); + } + + @Override + public JobState getState() { + return job.getState(); + } + + @Override + public Task getTask(TaskId taskId) { + return job.getTask(taskId); + } + + @Override + public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents( + int fromEventId, int maxEvents) { + return job.getTaskAttemptCompletionEvents(fromEventId, maxEvents); + } + + @Override + public Map getTasks() { + return job.getTasks(); + } + + @Override + protected void loadFullHistoryData(boolean loadTasks, + Path historyFileAbsolute) throws IOException { + //Empty + } + + @Override + public List getDiagnostics() { + return job.getDiagnostics(); + } + + @Override + public String getName() { + return job.getName(); + } + + @Override + public String getQueueName() { + return job.getQueueName(); + } + + @Override + public int getTotalMaps() { + return job.getTotalMaps(); + } + + @Override + public int getTotalReduces() { + return job.getTotalReduces(); + } + + @Override + public boolean isUber() { + return job.isUber(); + } + + @Override + public Map getTasks(TaskType taskType) { + return job.getTasks(); + } + + @Override + public + boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) { + return job.checkAccess(callerUGI, jobOperation); + } + + @Override + public Map getJobACLs() { + return job.getJobACLs(); + } + + @Override + public String getUserName() { + return job.getUserName(); + } + + @Override + public Path getConfFile() { + return job.getConfFile(); + } + + @Override + public List getAMInfos() { + return job.getAMInfos(); + } + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java index ec7df9be55..6267901f80 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; import java.io.StringReader; import java.util.Map; @@ -40,8 +41,11 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.MockJobs; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs; +import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.Clock; +import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.EventHandler; @@ -89,13 +93,21 @@ static class TestAppContext implements AppContext { final ApplicationAttemptId appAttemptID; final ApplicationId appID; final String user = MockJobs.newUserName(); - final Map jobs; + final Map partialJobs; + final Map fullJobs; final long startTime = System.currentTimeMillis(); TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); appAttemptID = MockJobs.newAppAttemptID(appID, 0); - jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); + JobsPair jobs; + try { + jobs = MockHistoryJobs.newHistoryJobs(appID, numJobs, numTasks, numAttempts); + } catch (IOException e) { + throw new YarnException(e); + } + partialJobs = jobs.partial; + fullJobs = jobs.full; } TestAppContext() { @@ -119,12 +131,16 @@ public CharSequence getUser() { @Override public Job getJob(JobId jobID) { - return jobs.get(jobID); + return fullJobs.get(jobID); } + public Job getPartialJob(JobId jobID) { + return partialJobs.get(jobID); + } + @Override public Map getAllJobs() { - return jobs; // OK + return partialJobs; // OK } @SuppressWarnings("rawtypes") @@ -204,8 +220,8 @@ public void testJobs() throws JSONException, Exception { JSONArray arr = jobs.getJSONArray("job"); assertEquals("incorrect number of elements", 1, arr.length()); JSONObject info = arr.getJSONObject(0); - Job job = appContext.getJob(MRApps.toJobID(info.getString("id"))); - VerifyJobsUtils.verifyHsJob(info, job); + Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); + VerifyJobsUtils.verifyHsJobPartial(info, job); } @@ -222,8 +238,8 @@ public void testJobsSlash() throws JSONException, Exception { JSONArray arr = jobs.getJSONArray("job"); assertEquals("incorrect number of elements", 1, arr.length()); JSONObject info = arr.getJSONObject(0); - Job job = appContext.getJob(MRApps.toJobID(info.getString("id"))); - VerifyJobsUtils.verifyHsJob(info, job); + Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); + VerifyJobsUtils.verifyHsJobPartial(info, job); } @@ -239,8 +255,8 @@ public void testJobsDefault() throws JSONException, Exception { JSONArray arr = jobs.getJSONArray("job"); assertEquals("incorrect number of elements", 1, arr.length()); JSONObject info = arr.getJSONObject(0); - Job job = appContext.getJob(MRApps.toJobID(info.getString("id"))); - VerifyJobsUtils.verifyHsJob(info, job); + Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); + VerifyJobsUtils.verifyHsJobPartial(info, job); } @@ -261,10 +277,35 @@ public void testJobsXML() throws Exception { assertEquals("incorrect number of elements", 1, jobs.getLength()); NodeList job = dom.getElementsByTagName("job"); assertEquals("incorrect number of elements", 1, job.getLength()); - verifyHsJobXML(job, appContext); - + verifyHsJobPartialXML(job, appContext); } + public void verifyHsJobPartialXML(NodeList nodes, TestAppContext appContext) { + + assertEquals("incorrect number of elements", 1, nodes.getLength()); + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + Job job = appContext.getPartialJob(MRApps.toJobID(WebServicesTestUtils + .getXmlString(element, "id"))); + assertNotNull("Job not found - output incorrect", job); + + VerifyJobsUtils.verifyHsJobGeneric(job, + WebServicesTestUtils.getXmlString(element, "id"), + WebServicesTestUtils.getXmlString(element, "user"), + WebServicesTestUtils.getXmlString(element, "name"), + WebServicesTestUtils.getXmlString(element, "state"), + WebServicesTestUtils.getXmlString(element, "queue"), + WebServicesTestUtils.getXmlLong(element, "startTime"), + WebServicesTestUtils.getXmlLong(element, "finishTime"), + WebServicesTestUtils.getXmlInt(element, "mapsTotal"), + WebServicesTestUtils.getXmlInt(element, "mapsCompleted"), + WebServicesTestUtils.getXmlInt(element, "reducesTotal"), + WebServicesTestUtils.getXmlInt(element, "reducesCompleted")); + } + } + public void verifyHsJobXML(NodeList nodes, TestAppContext appContext) { assertEquals("incorrect number of elements", 1, nodes.getLength()); @@ -320,7 +361,7 @@ public void testJobId() throws JSONException, Exception { JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("job"); - VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id)); + VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id)); } } @@ -356,7 +397,7 @@ public void testJobIdDefault() throws JSONException, Exception { JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("job"); - VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id)); + VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id)); } } @@ -694,7 +735,7 @@ public void testJobAttempts() throws JSONException, Exception { JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("jobAttempts"); - verifyHsJobAttempts(info, jobsMap.get(id)); + verifyHsJobAttempts(info, appContext.getJob(id)); } } @@ -712,7 +753,7 @@ public void testJobAttemptsSlash() throws JSONException, Exception { JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("jobAttempts"); - verifyHsJobAttempts(info, jobsMap.get(id)); + verifyHsJobAttempts(info, appContext.getJob(id)); } } @@ -730,7 +771,7 @@ public void testJobAttemptsDefault() throws JSONException, Exception { JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("jobAttempts"); - verifyHsJobAttempts(info, jobsMap.get(id)); + verifyHsJobAttempts(info, appContext.getJob(id)); } } @@ -754,7 +795,7 @@ public void testJobAttemptsXML() throws Exception { NodeList attempts = dom.getElementsByTagName("jobAttempts"); assertEquals("incorrect number of elements", 1, attempts.getLength()); NodeList info = dom.getElementsByTagName("jobAttempt"); - verifyHsJobAttemptsXML(info, jobsMap.get(id)); + verifyHsJobAttemptsXML(info, appContext.getJob(id)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java index 7376798e4c..0d9c83d37e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Map; @@ -34,8 +35,11 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.MockJobs; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs; +import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.Clock; +import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.EventHandler; @@ -73,11 +77,19 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { static class TestAppContext implements AppContext { final String user = MockJobs.newUserName(); - final Map jobs; + final Map fullJobs; + final Map partialJobs; final long startTime = System.currentTimeMillis(); TestAppContext(int numJobs, int numTasks, int numAttempts) { - jobs = MockJobs.newJobs(numJobs, numTasks, numAttempts); + JobsPair jobs; + try { + jobs = MockHistoryJobs.newHistoryJobs(numJobs, numTasks, numAttempts); + } catch (IOException e) { + throw new YarnException(e); + } + partialJobs = jobs.partial; + fullJobs = jobs.full; } TestAppContext() { @@ -101,12 +113,16 @@ public CharSequence getUser() { @Override public Job getJob(JobId jobID) { - return jobs.get(jobID); + return fullJobs.get(jobID); } + public Job getPartialJob(JobId jobID) { + return partialJobs.get(jobID); + } + @Override public Map getAllJobs() { - return jobs; // OK + return partialJobs; // OK } @SuppressWarnings("rawtypes") @@ -199,8 +215,8 @@ public void testJobsQueryUser() throws JSONException, Exception { assertEquals("incorrect number of elements", 3, arr.length()); // just verify one of them. JSONObject info = arr.getJSONObject(0); - Job job = appContext.getJob(MRApps.toJobID(info.getString("id"))); - VerifyJobsUtils.verifyHsJob(info, job); + Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); + VerifyJobsUtils.verifyHsJobPartial(info, job); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java index 1c99af9338..8820eeced2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java @@ -32,11 +32,20 @@ public class VerifyJobsUtils { - public static void verifyHsJob(JSONObject info, Job job) throws JSONException { + public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONException { + assertEquals("incorrect number of elements", 11, info.length()); - // this is 23 instead of 24 because acls not being checked since - // we are using mock job instead of CompletedJob - assertEquals("incorrect number of elements", 23, info.length()); + // everyone access fields + verifyHsJobGeneric(job, info.getString("id"), info.getString("user"), + info.getString("name"), info.getString("state"), + info.getString("queue"), info.getLong("startTime"), + info.getLong("finishTime"), info.getInt("mapsTotal"), + info.getInt("mapsCompleted"), info.getInt("reducesTotal"), + info.getInt("reducesCompleted")); + } + + public static void verifyHsJob(JSONObject info, Job job) throws JSONException { + assertEquals("incorrect number of elements", 24, info.length()); // everyone access fields verifyHsJobGeneric(job, info.getString("id"), info.getString("user"), diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm index f913bfc794..558b2c35ad 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -129,7 +129,7 @@ History Server REST API's. ** Jobs API - The jobs resource provides a list of the MapReduce jobs that have finished. + The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters *** URI @@ -160,7 +160,9 @@ History Server REST API's. *** Elements of the object When you make a request for the list of jobs, the information will be returned as an array of job objects. - See also {{Job API}} for syntax of the job object. + See also {{Job API}} for syntax of the job object. Except this is a subset of a full job. Only startTime, + finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are + returned. *---------------+--------------+-------------------------------+ || Item || Data Type || Description | @@ -194,73 +196,29 @@ History Server REST API's. "jobs" : { "job" : [ { - "avgReduceTime" : 833, - "failedReduceAttempts" : 0, "state" : "SUCCEEDED", - "successfulReduceAttempts" : 1, - "acls" : [ - { - "value" : " ", - "name" : "mapreduce.job.acl-modify-job" - }, - { - "value" : " ", - "name" : "mapreduce.job.acl-view-job" - } - ], "user" : "user1", "reducesTotal" : 1, "mapsCompleted" : 1, "startTime" : 1326381344489, "id" : "job_1326381300833_1_1", - "avgMapTime" : 2671, - "successfulMapAttempts" : 1, "name" : "word count", - "avgShuffleTime" : 2540, "reducesCompleted" : 1, - "diagnostics" : "", - "failedMapAttempts" : 0, - "avgMergeTime" : 2570, - "killedReduceAttempts" : 0, "mapsTotal" : 1, "queue" : "default", - "uberized" : false, - "killedMapAttempts" : 0, "finishTime" : 1326381356010 }, { - "avgReduceTime" : 124961, - "failedReduceAttempts" : 0, "state" : "SUCCEEDED", - "successfulReduceAttempts" : 1, - "acls" : [ - { - "value" : " ", - "name" : "mapreduce.job.acl-modify-job" - }, - { - "value" : " ", - "name" : "mapreduce.job.acl-view-job" - } - ], "user" : "user1", "reducesTotal" : 1, "mapsCompleted" : 1, "startTime" : 1326381446529, "id" : "job_1326381300833_2_2", - "avgMapTime" : 2638, - "successfulMapAttempts" : 1, "name" : "Sleep job", - "avgShuffleTime" : 2540, "reducesCompleted" : 1, - "diagnostics" : "", - "failedMapAttempts" : 0, - "avgMergeTime" : 2589, - "killedReduceAttempts" : 0, "mapsTotal" : 1, "queue" : "default", - "uberized" : false, - "killedMapAttempts" : 0, "finishTime" : 1326381582106 } ] @@ -303,26 +261,6 @@ History Server REST API's. 1 1 1 - false - - 2671 - 833 - 2540 - 2570 - 0 - 0 - 1 - 0 - 0 - 1 - - mapreduce.job.acl-modify-job - - - - mapreduce.job.acl-view-job - - 1326381446529 @@ -336,26 +274,6 @@ History Server REST API's. 1 1 1 - false - - 2638 - 124961 - 2540 - 2589 - 0 - 0 - 1 - 0 - 0 - 1 - - mapreduce.job.acl-modify-job - - - - mapreduce.job.acl-view-job - - +---+