big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2018-04-24 11:30:38 -07:00
parent 56788d759f
commit 1b9ecc264a
3 changed files with 80 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
View File

@ -148,7 +148,8 @@ private void printJobCounters(StringBuilder buff, Counters totalCounters,
"Total Value"));
buff.append("\n------------------------------------------" +
"---------------------------------------------");
for (String groupName : totalCounters.getGroupNames()) {
for (CounterGroup counterGroup : totalCounters) {
String groupName = counterGroup.getName();
CounterGroup totalGroup = totalCounters.getGroup(groupName);
CounterGroup mapGroup = mapCounters.getGroup(groupName);
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
View File

@ -104,7 +104,8 @@ private void printJobCounters(Counters totalCounters, Counters mapCounters,
// Killed jobs might not have counters
if (totalCounters != null) {
JSONObject jGroups = new JSONObject();
for (String groupName : totalCounters.getGroupNames()) {
for (CounterGroup counterGroup : totalCounters) {
String groupName = counterGroup.getName();
CounterGroup totalGroup = totalCounters.getGroup(groupName);
CounterGroup mapGroup = mapCounters.getGroup(groupName);
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);

76
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
View File

@ -859,6 +859,46 @@ public void testJSONPrinterAll() throws Exception {
"}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
}
@Test
public void testHumanDupePrinter() throws Exception {
JobHistoryParser.JobInfo job = createJobInfo2();
// Counters are only part of the overview so printAll can be false or true
// this does not affect the test, task counters are not printed
HumanReadableHistoryViewerPrinter printer =
new HumanReadableHistoryViewerPrinter(job, false, "http://",
TimeZone.getTimeZone("GMT"));
String outStr = run(printer);
// We are not interested in anything but the duplicate counter
int count1 = outStr.indexOf(
"|Map-Reduce Framework |Map input records |");
Assert.assertNotEquals("First counter occurrence not found", -1, count1);
int count2 = outStr.indexOf(
"|Map-Reduce Framework |Map input records |",
count1 + 1);
Assert.assertEquals("Duplicate counter found at: " + count1 +
" and " + count2, -1, count2);
}
@Test
public void testJSONDupePrinter() throws Exception {
JobHistoryParser.JobInfo job = createJobInfo2();
// Counters are part of the overview and task info
// Tasks only have bogus counters in the test if that is changed printAll
// must then be kept as false for this test to pass
JSONHistoryViewerPrinter printer =
new JSONHistoryViewerPrinter(job, false, "http://");
String outStr = run(printer);
// We are not interested in anything but the duplicate counter
int count1 = outStr.indexOf(
"\"counterName\":\"MAP_INPUT_RECORDS\"");
Assert.assertNotEquals("First counter occurrence not found", -1, count1);
int count2 = outStr.indexOf(
"\"counterName\":\"MAP_INPUT_RECORDS\"",
count1 + 1);
Assert.assertEquals("Duplicate counter found at: " + count1 +
" and " + count2, -1, count2);
}
private String run(HistoryViewerPrinter printer) throws Exception {
ByteArrayOutputStream boas = new ByteArrayOutputStream();
PrintStream out = new PrintStream(boas, true);
@ -901,6 +941,32 @@ private static JobHistoryParser.JobInfo createJobInfo() {
addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
return job;
}
private static JobHistoryParser.JobInfo createJobInfo2() {
JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
job.submitTime = 1317928501754L;
job.finishTime = job.submitTime + 15000;
job.jobid = JobID.forName("job_1317928501754_0001");
job.username = "test";
job.jobname = "Dupe counter output";
job.jobQueueName = "root.test";
job.jobConfPath = "/tmp/job.xml";
job.launchTime = job.submitTime + 1000;
job.totalMaps = 1;
job.totalReduces = 0;
job.failedMaps = 0;
job.failedReduces = 0;
job.succeededMaps = 1;
job.succeededReduces = 0;
job.jobStatus = JobStatus.State.SUCCEEDED.name();
job.totalCounters = createDeprecatedCounters();
job.mapCounters = createDeprecatedCounters();
job.reduceCounters = createDeprecatedCounters();
job.tasksMap = new HashMap<>();
addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED);
addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED);
return job;
}
private static Counters createCounters() {
Counters counters = new Counters();
@ -910,6 +976,16 @@ private static Counters createCounters() {
return counters;
}
private static Counters createDeprecatedCounters() {
Counters counters = new Counters();
// Deprecated counter: make sure it is only printed once
counters.findCounter("org.apache.hadoop.mapred.Task$Counter",
"MAP_INPUT_RECORDS").setValue(1);
counters.findCounter("File System Counters",
"FILE: Number of bytes read").setValue(1);
return counters;
}
private static void addTaskInfo(JobHistoryParser.JobInfo job,
TaskType type, int id, TaskStatus.State status) {
JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();