big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6579. JobStatus#getFailureInfo should not output diagnostic information when the job is running. (Akira AJISAKA via wangda)

Browse Source
This commit is contained in:
Wangda Tan 2016-03-15 17:28:40 -07:00
parent 5176a69868
commit 6529c87551
3 changed files with 84 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
View File

@ -171,7 +171,40 @@ public String toString() {
sb.append(blackListReport.replace("\n", ":")); sb.append(blackListReport.replace("\n", ":"));
return sb.toString(); return sb.toString();
} }
@Override
public int hashCode() {
int result = trackerName != null ? trackerName.hashCode() : 0;
result = 31 * result + (reasonForBlackListing != null ?
reasonForBlackListing.hashCode() : 0);
result = 31 * result + (blackListReport != null ?
blackListReport.hashCode() : 0);
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
final BlackListInfo that = (BlackListInfo) obj;
if (trackerName == null ? that.trackerName != null :
!trackerName.equals(that.trackerName)) {
return false;
}
if (reasonForBlackListing == null ? that.reasonForBlackListing != null :
!reasonForBlackListing.equals(that.reasonForBlackListing)) {
return false;
}
if (blackListReport == null ? that.blackListReport != null :
!blackListReport.equals(that.blackListReport)) {
return false;
}
return true;
}
} }
public static final long UNINITIALIZED_MEMORY_VALUE = -1; public static final long UNINITIALIZED_MEMORY_VALUE = -1;

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
View File

@ -137,7 +137,12 @@ public GetJobReportResponse getJobReport(GetJobReportRequest request)
jobReport.setJobState(jobState); jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser()); jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime()); jobReport.setStartTime(applicationReport.getStartTime());
jobReport.setDiagnostics(applicationReport.getDiagnostics()); YarnApplicationState state = applicationReport.getYarnApplicationState();
if (state == YarnApplicationState.KILLED
|| state == YarnApplicationState.FAILED
|| state == YarnApplicationState.FINISHED) {
jobReport.setDiagnostics(applicationReport.getDiagnostics());
}
jobReport.setJobName(applicationReport.getName()); jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl()); jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime()); jobReport.setFinishTime(applicationReport.getFinishTime());

89
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
View File

@ -155,9 +155,9 @@ public void testNetworkedJob() throws Exception {
// test getters // test getters
assertTrue(runningJob.getConfiguration().toString() assertTrue(runningJob.getConfiguration().toString()
.endsWith("0001/job.xml")); .endsWith("0001/job.xml"));
assertEquals(runningJob.getID(), jobId); assertEquals(jobId, runningJob.getID());
assertEquals(runningJob.getJobID(), jobId.toString()); assertEquals(jobId.toString(), runningJob.getJobID());
assertEquals(runningJob.getJobName(), "N/A"); assertEquals("N/A", runningJob.getJobName());
assertTrue(runningJob.getJobFile().endsWith( assertTrue(runningJob.getJobFile().endsWith(
".staging/" + runningJob.getJobID() + "/job.xml")); ".staging/" + runningJob.getJobID() + "/job.xml"));
assertTrue(runningJob.getTrackingURL().length() > 0); assertTrue(runningJob.getTrackingURL().length() > 0);
@ -169,40 +169,40 @@ public void testNetworkedJob() throws Exception {
TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0); TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0);
assertEquals(tce.length, 0); assertEquals(tce.length, 0);
assertEquals(runningJob.getHistoryUrl(),""); assertEquals("", runningJob.getHistoryUrl());
assertFalse(runningJob.isRetired()); assertFalse(runningJob.isRetired());
assertEquals( runningJob.getFailureInfo(),""); assertEquals("", runningJob.getFailureInfo());
assertEquals(runningJob.getJobStatus().getJobName(), "N/A"); assertEquals("N/A", runningJob.getJobStatus().getJobName());
assertEquals(client.getMapTaskReports(jobId).length, 0); assertEquals(0, client.getMapTaskReports(jobId).length);
try { try {
client.getSetupTaskReports(jobId); client.getSetupTaskReports(jobId);
} catch (YarnRuntimeException e) { } catch (YarnRuntimeException e) {
assertEquals(e.getMessage(), "Unrecognized task type: JOB_SETUP"); assertEquals("Unrecognized task type: JOB_SETUP", e.getMessage());
} }
try { try {
client.getCleanupTaskReports(jobId); client.getCleanupTaskReports(jobId);
} catch (YarnRuntimeException e) { } catch (YarnRuntimeException e) {
assertEquals(e.getMessage(), "Unrecognized task type: JOB_CLEANUP"); assertEquals("Unrecognized task type: JOB_CLEANUP", e.getMessage());
} }
assertEquals(client.getReduceTaskReports(jobId).length, 0); assertEquals(0, client.getReduceTaskReports(jobId).length);
// test ClusterStatus // test ClusterStatus
ClusterStatus status = client.getClusterStatus(true); ClusterStatus status = client.getClusterStatus(true);
assertEquals(status.getActiveTrackerNames().size(), 2); assertEquals(2, status.getActiveTrackerNames().size());
// it method does not implemented and always return empty array or null; // it method does not implemented and always return empty array or null;
assertEquals(status.getBlacklistedTrackers(), 0); assertEquals(0, status.getBlacklistedTrackers());
assertEquals(status.getBlacklistedTrackerNames().size(), 0); assertEquals(0, status.getBlacklistedTrackerNames().size());
assertEquals(status.getBlackListedTrackersInfo().size(), 0); assertEquals(0, status.getBlackListedTrackersInfo().size());
assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus());
assertEquals(status.getMapTasks(), 1); assertEquals(1, status.getMapTasks());
assertEquals(status.getMaxMapTasks(), 20); assertEquals(20, status.getMaxMapTasks());
assertEquals(status.getMaxReduceTasks(), 4); assertEquals(4, status.getMaxReduceTasks());
assertEquals(status.getNumExcludedNodes(), 0); assertEquals(0, status.getNumExcludedNodes());
assertEquals(status.getReduceTasks(), 1); assertEquals(1, status.getReduceTasks());
assertEquals(status.getTaskTrackers(), 2); assertEquals(2, status.getTaskTrackers());
assertEquals(status.getTTExpiryInterval(), 0); assertEquals(0, status.getTTExpiryInterval());
assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus());
assertEquals(status.getGraylistedTrackers(), 0); assertEquals(0, status.getGraylistedTrackers());
// test read and write // test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
@ -219,34 +219,34 @@ public void testNetworkedJob() throws Exception {
// test taskStatusfilter // test taskStatusfilter
JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL); JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL);
assertEquals(JobClient.getTaskOutputFilter(job), TaskStatusFilter.ALL); assertEquals(TaskStatusFilter.ALL, JobClient.getTaskOutputFilter(job));
// runningJob.setJobPriority(JobPriority.HIGH.name()); // runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map // test default map
assertEquals(client.getDefaultMaps(), 20); assertEquals(20, client.getDefaultMaps());
assertEquals(client.getDefaultReduces(), 4); assertEquals(4, client.getDefaultReduces());
assertEquals(client.getSystemDir().getName(), "jobSubmitDir"); assertEquals("jobSubmitDir", client.getSystemDir().getName());
// test queue information // test queue information
JobQueueInfo[] rootQueueInfo = client.getRootQueues(); JobQueueInfo[] rootQueueInfo = client.getRootQueues();
assertEquals(rootQueueInfo.length, 1); assertEquals(1, rootQueueInfo.length);
assertEquals(rootQueueInfo[0].getQueueName(), "default"); assertEquals("default", rootQueueInfo[0].getQueueName());
JobQueueInfo[] qinfo = client.getQueues(); JobQueueInfo[] qinfo = client.getQueues();
assertEquals(qinfo.length, 1); assertEquals(1, qinfo.length);
assertEquals(qinfo[0].getQueueName(), "default"); assertEquals("default", qinfo[0].getQueueName());
assertEquals(client.getChildQueues("default").length, 0); assertEquals(0, client.getChildQueues("default").length);
assertEquals(client.getJobsFromQueue("default").length, 1); assertEquals(1, client.getJobsFromQueue("default").length);
assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith( assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith(
"/job.xml")); "/job.xml"));
JobQueueInfo qi = client.getQueueInfo("default"); JobQueueInfo qi = client.getQueueInfo("default");
assertEquals(qi.getQueueName(), "default"); assertEquals("default", qi.getQueueName());
assertEquals(qi.getQueueState(), "running"); assertEquals("running", qi.getQueueState());
QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser(); QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser();
assertEquals(aai.length, 2); assertEquals(2, aai.length);
assertEquals(aai[0].getQueueName(), "root"); assertEquals("root", aai[0].getQueueName());
assertEquals(aai[1].getQueueName(), "default"); assertEquals("default", aai[1].getQueueName());
// test JobClient // test JobClient
// The following asserts read JobStatus twice and ensure the returned // The following asserts read JobStatus twice and ensure the returned
@ -282,12 +282,11 @@ public void testBlackListInfo() throws IOException {
BlackListInfo info2 = new BlackListInfo(); BlackListInfo info2 = new BlackListInfo();
info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut
.toByteArray()))); .toByteArray())));
assertEquals(info, info); assertEquals(info, info2);
assertEquals(info.toString(), info.toString()); assertEquals(info.toString(), info2.toString());
assertEquals(info.getTrackerName(), "trackerName"); assertEquals("trackerName", info2.getTrackerName());
assertEquals(info.getReasonForBlackListing(), "reasonForBlackListing"); assertEquals("reasonForBlackListing", info2.getReasonForBlackListing());
assertEquals(info.getBlackListReport(), "blackListInfo"); assertEquals("blackListInfo", info2.getBlackListReport());
} }
/** /**
* test run from command line JobQueueClient * test run from command line JobQueueClient