diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 115fc1b903..3c1884dd03 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -465,6 +465,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes (Ray Chiang via ozawa). + MAPREDUCE-6350. JobHistory doesn't support fully-functional search. + (Siqi Li via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index bf32888098..35556a698c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; import org.apache.hadoop.security.UserGroupInformation; @@ -1107,9 +1108,12 @@ protected void processDoneFiles(JobId jobId) throws IOException { if (mi.getHistoryFile() != null) { Path historyFile = mi.getHistoryFile(); Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile); + int jobNameLimit = + getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT, + JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT); String doneJobHistoryFileName = getTempFileName(FileNameIndexUtils.getDoneFileName(mi - .getJobIndexInfo())); + .getJobIndexInfo(), jobNameLimit)); qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, doneJobHistoryFileName)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java index 741da11b4a..eb0c54c1ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java @@ -32,8 +32,6 @@ public class FileNameIndexUtils { - static final int JOB_NAME_TRIM_LENGTH = 50; - // Sanitize job history file for predictable parsing static final String DELIMITER = "-"; static final String DELIMITER_ESCAPE = "%2D"; @@ -60,6 +58,12 @@ public class FileNameIndexUtils { * @return the done job history filename. */ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException { + return getDoneFileName(indexInfo, + JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT); + } + + public static String getDoneFileName(JobIndexInfo indexInfo, + int jobNameLimit) throws IOException { StringBuilder sb = new StringBuilder(); //JobId sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString())); @@ -74,7 +78,8 @@ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException sb.append(DELIMITER); //JobName - sb.append(escapeDelimiters(trimJobName(getJobName(indexInfo)))); + sb.append(escapeDelimiters(trimJobName( + getJobName(indexInfo), jobNameLimit))); sb.append(DELIMITER); //FinishTime @@ -286,9 +291,9 @@ private static String escapeDelimiters(String escapee) { /** * Trims the job-name if required */ - private static String trimJobName(String jobName) { - if (jobName.length() > JOB_NAME_TRIM_LENGTH) { - jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH); + private static String trimJobName(String jobName, int jobNameLimit) { + if (jobName.length() > jobNameLimit) { + jobName = jobName.substring(0, jobNameLimit); } return jobName; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index f7cba9f806..a97c2ca9e7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -214,4 +214,11 @@ public class JHAdminConfig { */ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false; + /** + * Number of characters allowed for job name in Job History Server web page. + */ + public static final String MR_HS_JOBNAME_LIMIT = MR_HISTORY_PREFIX + + "jobname.limit"; + public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50; + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java index a0d5fceb35..3d722e00e8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java @@ -147,6 +147,32 @@ public void testUserNamePercentEncoding() throws IOException { jobHistoryFile.contains(USER_NAME_WITH_DELIMITER_ESCAPE)); } + @Test + public void testTrimJobName() throws IOException { + int jobNameTrimLength = 5; + JobIndexInfo info = new JobIndexInfo(); + JobID oldJobId = JobID.forName(JOB_ID); + JobId jobId = TypeConverter.toYarn(oldJobId); + info.setJobId(jobId); + info.setSubmitTime(Long.parseLong(SUBMIT_TIME)); + info.setUser(USER_NAME); + info.setJobName(JOB_NAME); + info.setFinishTime(Long.parseLong(FINISH_TIME)); + info.setNumMaps(Integer.parseInt(NUM_MAPS)); + info.setNumReduces(Integer.parseInt(NUM_REDUCES)); + info.setJobStatus(JOB_STATUS); + info.setQueueName(QUEUE_NAME); + info.setJobStartTime(Long.parseLong(JOB_START_TIME)); + + String jobHistoryFile = + FileNameIndexUtils.getDoneFileName(info, jobNameTrimLength); + JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile); + + Assert.assertEquals("Job name did not get trimmed correctly", + info.getJobName().substring(0, jobNameTrimLength), + parsedInfo.getJobName()); + } + @Test public void testUserNamePercentDecoding() throws IOException { String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 4a2dd7717a..3789429eb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1705,6 +1705,14 @@ + + mapreduce.jobhistory.jobname.limit + 50 + + Number of characters allowed for job name in Job History Server web page. + + + mapreduce.job.heap.memory-mb.ratio 0.8