big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3944. Change the history jobs/ webservice to return partial job info for a significant performance improvement. (Contributed by Robert Joseph Evans)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1297672 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Siddharth Seth 2012-03-06 19:52:00 +00:00
parent 1a0e088a19
commit 4ce5f6553f
9 changed files with 352 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -176,6 +176,10 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-2855. Passing a cached class-loader to ResourceBundle creator to
minimize counter names lookup time. (Siddarth Seth via vinodkv)
MAPREDUCE-3944. Change the history jobs/ webservice to return partial job
info for a significant performance improvement. (Robert Joseph Evans via
sseth)
BUG FIXES
MAPREDUCE-3918 proc_historyserver no longer in command line arguments for
HistoryServer (Jon Eagles via bobby)

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
View File

@ -287,7 +287,7 @@ private void loadAllTasks() {
}
//History data is leisurely loaded when task level data is requested
private synchronized void loadFullHistoryData(boolean loadTasks,
protected synchronized void loadFullHistoryData(boolean loadTasks,
Path historyFileAbsolute) throws IOException {
LOG.info("Loading history file: [" + historyFileAbsolute + "]");
if (this.jobInfo != null) {

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
@ -184,35 +185,32 @@ public JobsInfo getJobs(@QueryParam("user") String userQuery,
break;
}
// getAllJobs only gives you a partial we want a full
Job fullJob = appCtx.getJob(job.getID());
if (fullJob == null) {
continue;
}
JobInfo jobInfo = new JobInfo(fullJob);
// can't really validate queue is a valid one since queues could change
if (queueQuery != null && !queueQuery.isEmpty()) {
if (!jobInfo.getQueueName().equals(queueQuery)) {
if (!job.getQueueName().equals(queueQuery)) {
continue;
}
}
if (userQuery != null && !userQuery.isEmpty()) {
if (!jobInfo.getUserName().equals(userQuery)) {
if (!job.getUserName().equals(userQuery)) {
continue;
}
}
JobReport report = job.getReport();
if (checkStart
&& (jobInfo.getStartTime() < sBegin || jobInfo.getStartTime() > sEnd)) {
&& (report.getStartTime() < sBegin || report.getStartTime() > sEnd)) {
continue;
}
if (checkEnd
&& (jobInfo.getFinishTime() < fBegin || jobInfo.getFinishTime() > fEnd)) {
&& (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd)) {
continue;
}
JobInfo jobInfo = new JobInfo(job);
allJobs.add(jobInfo);
num++;
}

80
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
View File

@ -55,18 +55,18 @@ public class JobInfo {
protected int mapsCompleted;
protected int reducesTotal;
protected int reducesCompleted;
protected boolean uberized;
protected Boolean uberized;
protected String diagnostics;
protected long avgMapTime = 0;
protected long avgReduceTime = 0;
protected long avgShuffleTime = 0;
protected long avgMergeTime = 0;
protected int failedReduceAttempts = 0;
protected int killedReduceAttempts = 0;
protected int successfulReduceAttempts = 0;
protected int failedMapAttempts = 0;
protected int killedMapAttempts = 0;
protected int successfulMapAttempts = 0;
protected Long avgMapTime;
protected Long avgReduceTime;
protected Long avgShuffleTime;
protected Long avgMergeTime;
protected Integer failedReduceAttempts;
protected Integer killedReduceAttempts;
protected Integer successfulReduceAttempts;
protected Integer failedMapAttempts;
protected Integer killedMapAttempts;
protected Integer successfulMapAttempts;
protected ArrayList<ConfEntryInfo> acls;
@XmlTransient
@ -80,7 +80,7 @@ public JobInfo() {
public JobInfo(Job job) {
this.id = MRApps.toString(job.getID());
JobReport report = job.getReport();
countTasksAndAttempts(job);
this.mapsTotal = job.getTotalMaps();
this.mapsCompleted = job.getCompletedMaps();
this.reducesTotal = job.getTotalReduces();
@ -91,19 +91,33 @@ public JobInfo(Job job) {
this.queue = job.getQueueName();
this.user = job.getUserName();
this.state = job.getState().toString();
this.uberized = job.isUber();
this.diagnostics = "";
List<String> diagnostics = job.getDiagnostics();
if (diagnostics != null && !diagnostics.isEmpty()) {
StringBuffer b = new StringBuffer();
for (String diag : diagnostics) {
b.append(diag);
}
this.diagnostics = b.toString();
}
this.acls = new ArrayList<ConfEntryInfo>();
if (job instanceof CompletedJob) {
avgMapTime = 0l;
avgReduceTime = 0l;
avgShuffleTime = 0l;
avgMergeTime = 0l;
failedReduceAttempts = 0;
killedReduceAttempts = 0;
successfulReduceAttempts = 0;
failedMapAttempts = 0;
killedMapAttempts = 0;
successfulMapAttempts = 0;
countTasksAndAttempts(job);
this.uberized = job.isUber();
this.diagnostics = "";
List<String> diagnostics = job.getDiagnostics();
if (diagnostics != null && !diagnostics.isEmpty()) {
StringBuffer b = new StringBuffer();
for (String diag : diagnostics) {
b.append(diag);
}
this.diagnostics = b.toString();
}
Map<JobACL, AccessControlList> allacls = job.getJobACLs();
if (allacls != null) {
for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
@ -122,43 +136,43 @@ public long getNumReduces() {
return numReduces;
}
public long getAvgMapTime() {
public Long getAvgMapTime() {
return avgMapTime;
}
public long getAvgReduceTime() {
public Long getAvgReduceTime() {
return avgReduceTime;
}
public long getAvgShuffleTime() {
public Long getAvgShuffleTime() {
return avgShuffleTime;
}
public long getAvgMergeTime() {
public Long getAvgMergeTime() {
return avgMergeTime;
}
public long getFailedReduceAttempts() {
public Integer getFailedReduceAttempts() {
return failedReduceAttempts;
}
public long getKilledReduceAttempts() {
public Integer getKilledReduceAttempts() {
return killedReduceAttempts;
}
public long getSuccessfulReduceAttempts() {
public Integer getSuccessfulReduceAttempts() {
return successfulReduceAttempts;
}
public long getFailedMapAttempts() {
public Integer getFailedMapAttempts() {
return failedMapAttempts;
}
public long getKilledMapAttempts() {
public Integer getKilledMapAttempts() {
return killedMapAttempts;
}
public long getSuccessfulMapAttempts() {
public Integer getSuccessfulMapAttempts() {
return successfulMapAttempts;
}
@ -210,7 +224,7 @@ public long getFinishTime() {
return this.finishTime;
}
public boolean isUber() {
public Boolean isUber() {
return this.uberized;
}

192
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java Normal file
View File

@ -0,0 +1,192 @@
package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import com.google.common.collect.Maps;
public class MockHistoryJobs extends MockJobs {
public static class JobsPair {
public Map<JobId, Job> partial;
public Map<JobId, Job> full;
}
public static JobsPair newHistoryJobs(int numJobs, int numTasksPerJob,
int numAttemptsPerTask) throws IOException {
Map<JobId, Job> mocked = newJobs(numJobs, numTasksPerJob, numAttemptsPerTask);
return split(mocked);
}
public static JobsPair newHistoryJobs(ApplicationId appID, int numJobsPerApp,
int numTasksPerJob, int numAttemptsPerTask) throws IOException {
Map<JobId, Job> mocked = newJobs(appID, numJobsPerApp, numTasksPerJob,
numAttemptsPerTask);
return split(mocked);
}
private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
JobsPair ret = new JobsPair();
ret.full = Maps.newHashMap();
ret.partial = Maps.newHashMap();
for(Map.Entry<JobId, Job> entry: mocked.entrySet()) {
JobId id = entry.getKey();
Job j = entry.getValue();
ret.full.put(id, new MockCompletedJob(j));
JobReport report = j.getReport();
JobIndexInfo info = new JobIndexInfo(report.getStartTime(),
report.getFinishTime(), j.getUserName(), j.getName(), id,
j.getCompletedMaps(), j.getCompletedReduces(), String.valueOf(j.getState()));
info.setQueueName(j.getQueueName());
ret.partial.put(id, new PartialJob(info, id));
}
return ret;
}
private static class MockCompletedJob extends CompletedJob {
private Job job;
public MockCompletedJob(Job job) throws IOException {
super(new Configuration(), job.getID(), null, true, job.getUserName(),
null, null);
this.job = job;
}
@Override
public int getCompletedMaps() {
return job.getCompletedMaps();
}
@Override
public int getCompletedReduces() {
return job.getCompletedReduces();
}
@Override
public org.apache.hadoop.mapreduce.Counters getAllCounters() {
return job.getAllCounters();
}
@Override
public JobId getID() {
return job.getID();
}
@Override
public JobReport getReport() {
return job.getReport();
}
@Override
public float getProgress() {
return job.getProgress();
}
@Override
public JobState getState() {
return job.getState();
}
@Override
public Task getTask(TaskId taskId) {
return job.getTask(taskId);
}
@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
int fromEventId, int maxEvents) {
return job.getTaskAttemptCompletionEvents(fromEventId, maxEvents);
}
@Override
public Map<TaskId, Task> getTasks() {
return job.getTasks();
}
@Override
protected void loadFullHistoryData(boolean loadTasks,
Path historyFileAbsolute) throws IOException {
//Empty
}
@Override
public List<String> getDiagnostics() {
return job.getDiagnostics();
}
@Override
public String getName() {
return job.getName();
}
@Override
public String getQueueName() {
return job.getQueueName();
}
@Override
public int getTotalMaps() {
return job.getTotalMaps();
}
@Override
public int getTotalReduces() {
return job.getTotalReduces();
}
@Override
public boolean isUber() {
return job.isUber();
}
@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
return job.getTasks();
}
@Override
public
boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) {
return job.checkAccess(callerUGI, jobOperation);
}
@Override
public Map<JobACL, AccessControlList> getJobACLs() {
return job.getJobACLs();
}
@Override
public String getUserName() {
return job.getUserName();
}
@Override
public Path getConfFile() {
return job.getConfFile();
}
@Override
public List<AMInfo> getAMInfos() {
return job.getAMInfos();
}
}
}

77
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
View File

@ -27,6 +27,7 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
@ -40,8 +41,11 @@
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
@ -89,13 +93,21 @@ static class TestAppContext implements AppContext {
final ApplicationAttemptId appAttemptID;
final ApplicationId appID;
final String user = MockJobs.newUserName();
final Map<JobId, Job> jobs;
final Map<JobId, Job> partialJobs;
final Map<JobId, Job> fullJobs;
final long startTime = System.currentTimeMillis();
TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
appID = MockJobs.newAppID(appid);
appAttemptID = MockJobs.newAppAttemptID(appID, 0);
jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
JobsPair jobs;
try {
jobs = MockHistoryJobs.newHistoryJobs(appID, numJobs, numTasks, numAttempts);
} catch (IOException e) {
throw new YarnException(e);
}
partialJobs = jobs.partial;
fullJobs = jobs.full;
}
TestAppContext() {
@ -119,12 +131,16 @@ public CharSequence getUser() {
@Override
public Job getJob(JobId jobID) {
return jobs.get(jobID);
return fullJobs.get(jobID);
}
public Job getPartialJob(JobId jobID) {
return partialJobs.get(jobID);
}
@Override
public Map<JobId, Job> getAllJobs() {
return jobs; // OK
return partialJobs; // OK
}
@SuppressWarnings("rawtypes")
@ -204,8 +220,8 @@ public void testJobs() throws JSONException, Exception {
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJob(info, job);
Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@ -222,8 +238,8 @@ public void testJobsSlash() throws JSONException, Exception {
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJob(info, job);
Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@ -239,8 +255,8 @@ public void testJobsDefault() throws JSONException, Exception {
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJob(info, job);
Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@ -261,10 +277,35 @@ public void testJobsXML() throws Exception {
assertEquals("incorrect number of elements", 1, jobs.getLength());
NodeList job = dom.getElementsByTagName("job");
assertEquals("incorrect number of elements", 1, job.getLength());
verifyHsJobXML(job, appContext);
verifyHsJobPartialXML(job, appContext);
}
public void verifyHsJobPartialXML(NodeList nodes, TestAppContext appContext) {
assertEquals("incorrect number of elements", 1, nodes.getLength());
for (int i = 0; i < nodes.getLength(); i++) {
Element element = (Element) nodes.item(i);
Job job = appContext.getPartialJob(MRApps.toJobID(WebServicesTestUtils
.getXmlString(element, "id")));
assertNotNull("Job not found - output incorrect", job);
VerifyJobsUtils.verifyHsJobGeneric(job,
WebServicesTestUtils.getXmlString(element, "id"),
WebServicesTestUtils.getXmlString(element, "user"),
WebServicesTestUtils.getXmlString(element, "name"),
WebServicesTestUtils.getXmlString(element, "state"),
WebServicesTestUtils.getXmlString(element, "queue"),
WebServicesTestUtils.getXmlLong(element, "startTime"),
WebServicesTestUtils.getXmlLong(element, "finishTime"),
WebServicesTestUtils.getXmlInt(element, "mapsTotal"),
WebServicesTestUtils.getXmlInt(element, "mapsCompleted"),
WebServicesTestUtils.getXmlInt(element, "reducesTotal"),
WebServicesTestUtils.getXmlInt(element, "reducesCompleted"));
}
}
public void verifyHsJobXML(NodeList nodes, TestAppContext appContext) {
assertEquals("incorrect number of elements", 1, nodes.getLength());
@ -320,7 +361,7 @@ public void testJobId() throws JSONException, Exception {
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("job");
VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id));
VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id));
}
}
@ -356,7 +397,7 @@ public void testJobIdDefault() throws JSONException, Exception {
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("job");
VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id));
VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id));
}
}
@ -694,7 +735,7 @@ public void testJobAttempts() throws JSONException, Exception {
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
verifyHsJobAttempts(info, jobsMap.get(id));
verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@ -712,7 +753,7 @@ public void testJobAttemptsSlash() throws JSONException, Exception {
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
verifyHsJobAttempts(info, jobsMap.get(id));
verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@ -730,7 +771,7 @@ public void testJobAttemptsDefault() throws JSONException, Exception {
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
verifyHsJobAttempts(info, jobsMap.get(id));
verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@ -754,7 +795,7 @@ public void testJobAttemptsXML() throws Exception {
NodeList attempts = dom.getElementsByTagName("jobAttempts");
assertEquals("incorrect number of elements", 1, attempts.getLength());
NodeList info = dom.getElementsByTagName("jobAttempt");
verifyHsJobAttemptsXML(info, jobsMap.get(id));
verifyHsJobAttemptsXML(info, appContext.getJob(id));
}
}

28
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
View File

@ -23,6 +23,7 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;
@ -34,8 +35,11 @@
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs;
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
@ -73,11 +77,19 @@ public class TestHsWebServicesJobsQuery extends JerseyTest {
static class TestAppContext implements AppContext {
final String user = MockJobs.newUserName();
final Map<JobId, Job> jobs;
final Map<JobId, Job> fullJobs;
final Map<JobId, Job> partialJobs;
final long startTime = System.currentTimeMillis();
TestAppContext(int numJobs, int numTasks, int numAttempts) {
jobs = MockJobs.newJobs(numJobs, numTasks, numAttempts);
JobsPair jobs;
try {
jobs = MockHistoryJobs.newHistoryJobs(numJobs, numTasks, numAttempts);
} catch (IOException e) {
throw new YarnException(e);
}
partialJobs = jobs.partial;
fullJobs = jobs.full;
}
TestAppContext() {
@ -101,12 +113,16 @@ public CharSequence getUser() {
@Override
public Job getJob(JobId jobID) {
return jobs.get(jobID);
return fullJobs.get(jobID);
}
public Job getPartialJob(JobId jobID) {
return partialJobs.get(jobID);
}
@Override
public Map<JobId, Job> getAllJobs() {
return jobs; // OK
return partialJobs; // OK
}
@SuppressWarnings("rawtypes")
@ -199,8 +215,8 @@ public void testJobsQueryUser() throws JSONException, Exception {
assertEquals("incorrect number of elements", 3, arr.length());
// just verify one of them.
JSONObject info = arr.getJSONObject(0);
Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJob(info, job);
Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@Test

17
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java
View File

@ -32,11 +32,20 @@
public class VerifyJobsUtils {
public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONException {
assertEquals("incorrect number of elements", 11, info.length());
// this is 23 instead of 24 because acls not being checked since
// we are using mock job instead of CompletedJob
assertEquals("incorrect number of elements", 23, info.length());
// everyone access fields
verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("state"),
info.getString("queue"), info.getLong("startTime"),
info.getLong("finishTime"), info.getInt("mapsTotal"),
info.getInt("mapsCompleted"), info.getInt("reducesTotal"),
info.getInt("reducesCompleted"));
}
public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
assertEquals("incorrect number of elements", 24, info.length());
// everyone access fields
verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),

90
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
View File

@ -129,7 +129,7 @@ History Server REST API's.
** Jobs API
The jobs resource provides a list of the MapReduce jobs that have finished.
The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters
*** URI
@ -160,7 +160,9 @@ History Server REST API's.
*** Elements of the <jobs> object
When you make a request for the list of jobs, the information will be returned as an array of job objects.
See also {{Job API}} for syntax of the job object.
See also {{Job API}} for syntax of the job object. Except this is a subset of a full job. Only startTime,
finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are
returned.
*---------------+--------------+-------------------------------+
|| Item || Data Type || Description |
@ -194,73 +196,29 @@ History Server REST API's.
"jobs" : {
"job" : [
{
"avgReduceTime" : 833,
"failedReduceAttempts" : 0,
"state" : "SUCCEEDED",
"successfulReduceAttempts" : 1,
"acls" : [
{
"value" : " ",
"name" : "mapreduce.job.acl-modify-job"
},
{
"value" : " ",
"name" : "mapreduce.job.acl-view-job"
}
],
"user" : "user1",
"reducesTotal" : 1,
"mapsCompleted" : 1,
"startTime" : 1326381344489,
"id" : "job_1326381300833_1_1",
"avgMapTime" : 2671,
"successfulMapAttempts" : 1,
"name" : "word count",
"avgShuffleTime" : 2540,
"reducesCompleted" : 1,
"diagnostics" : "",
"failedMapAttempts" : 0,
"avgMergeTime" : 2570,
"killedReduceAttempts" : 0,
"mapsTotal" : 1,
"queue" : "default",
"uberized" : false,
"killedMapAttempts" : 0,
"finishTime" : 1326381356010
},
{
"avgReduceTime" : 124961,
"failedReduceAttempts" : 0,
"state" : "SUCCEEDED",
"successfulReduceAttempts" : 1,
"acls" : [
{
"value" : " ",
"name" : "mapreduce.job.acl-modify-job"
},
{
"value" : " ",
"name" : "mapreduce.job.acl-view-job"
}
],
"user" : "user1",
"reducesTotal" : 1,
"mapsCompleted" : 1,
"startTime" : 1326381446529,
"id" : "job_1326381300833_2_2",
"avgMapTime" : 2638,
"successfulMapAttempts" : 1,
"name" : "Sleep job",
"avgShuffleTime" : 2540,
"reducesCompleted" : 1,
"diagnostics" : "",
"failedMapAttempts" : 0,
"avgMergeTime" : 2589,
"killedReduceAttempts" : 0,
"mapsTotal" : 1,
"queue" : "default",
"uberized" : false,
"killedMapAttempts" : 0,
"finishTime" : 1326381582106
}
]
@ -303,26 +261,6 @@ History Server REST API's.
<mapsCompleted>1</mapsCompleted>
<reducesTotal>1</reducesTotal>
<reducesCompleted>1</reducesCompleted>
<uberized>false</uberized>
<diagnostics/>
<avgMapTime>2671</avgMapTime>
<avgReduceTime>833</avgReduceTime>
<avgShuffleTime>2540</avgShuffleTime>
<avgMergeTime>2570</avgMergeTime>
<failedReduceAttempts>0</failedReduceAttempts>
<killedReduceAttempts>0</killedReduceAttempts>
<successfulReduceAttempts>1</successfulReduceAttempts>
<failedMapAttempts>0</failedMapAttempts>
<killedMapAttempts>0</killedMapAttempts>
<successfulMapAttempts>1</successfulMapAttempts>
<acls>
<name>mapreduce.job.acl-modify-job</name>
<value> </value>
</acls>
<acls>
<name>mapreduce.job.acl-view-job</name>
<value> </value>
</acls>
</job>
<job>
<startTime>1326381446529</startTime>
@ -336,26 +274,6 @@ History Server REST API's.
<mapsCompleted>1</mapsCompleted>
<reducesTotal>1</reducesTotal>
<reducesCompleted>1</reducesCompleted>
<uberized>false</uberized>
<diagnostics/>
<avgMapTime>2638</avgMapTime>
<avgReduceTime>124961</avgReduceTime>
<avgShuffleTime>2540</avgShuffleTime>
<avgMergeTime>2589</avgMergeTime>
<failedReduceAttempts>0</failedReduceAttempts>
<killedReduceAttempts>0</killedReduceAttempts>
<successfulReduceAttempts>1</successfulReduceAttempts>
<failedMapAttempts>0</failedMapAttempts>
<killedMapAttempts>0</killedMapAttempts>
<successfulMapAttempts>1</successfulMapAttempts>
<acls>
<name>mapreduce.job.acl-modify-job</name>
<value> </value>
</acls>
<acls>
<name>mapreduce.job.acl-view-job</name>
<value> </value>
</acls>
</job>
</jobs>
+---+