big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape hyphens and thus avoid parsing errors. Contributed by Akira Ajisaka.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1582125 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-03-26 23:43:49 +00:00
parent 82be0f935d
commit 533675bfbb
3 changed files with 48 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -266,6 +266,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5795. Fixed MRAppMaster to record the correct job-state after it
recovers from a commit during a previous attempt. (Xuan Gong via vinodkv)
MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape
hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv)
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
View File

@ -94,7 +94,7 @@ public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException
sb.append(DELIMITER);
//QueueName
sb.append(indexInfo.getQueueName());
sb.append(escapeDelimiters(indexInfo.getQueueName()));
sb.append(DELIMITER);
//JobStartTime

44
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
View File

@ -68,6 +68,10 @@ public class TestFileNameIndexUtils {
private static final String NUM_REDUCES = "1";
private static final String JOB_STATUS = "SUCCEEDED";
private static final String QUEUE_NAME = "default";
private static final String QUEUE_NAME_WITH_DELIMITER = "test"
+ FileNameIndexUtils.DELIMITER + "queue";
private static final String QUEUE_NAME_WITH_DELIMITER_ESCAPE = "test"
+ FileNameIndexUtils.DELIMITER_ESCAPE + "queue";
private static final String JOB_START_TIME = "1317928742060";
@Test
@ -191,6 +195,46 @@ public void testJobNamePercentDecoding() throws IOException {
JOB_NAME_WITH_DELIMITER, info.getJobName());
}
@Test
public void testQueueNamePercentEncoding() throws IOException {
JobIndexInfo info = new JobIndexInfo();
JobID oldJobId = JobID.forName(JOB_ID);
JobId jobId = TypeConverter.toYarn(oldJobId);
info.setJobId(jobId);
info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
info.setUser(USER_NAME);
info.setJobName(JOB_NAME);
info.setFinishTime(Long.parseLong(FINISH_TIME));
info.setNumMaps(Integer.parseInt(NUM_MAPS));
info.setNumReduces(Integer.parseInt(NUM_REDUCES));
info.setJobStatus(JOB_STATUS);
info.setQueueName(QUEUE_NAME_WITH_DELIMITER);
info.setJobStartTime(Long.parseLong(JOB_START_TIME));
String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
Assert.assertTrue("Queue name not encoded correctly into job history file",
jobHistoryFile.contains(QUEUE_NAME_WITH_DELIMITER_ESCAPE));
}
@Test
public void testQueueNamePercentDecoding() throws IOException {
String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER,
JOB_ID,
SUBMIT_TIME,
USER_NAME,
JOB_NAME,
FINISH_TIME,
NUM_MAPS,
NUM_REDUCES,
JOB_STATUS,
QUEUE_NAME_WITH_DELIMITER_ESCAPE,
JOB_START_TIME );
JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
Assert.assertEquals("Queue name doesn't match",
QUEUE_NAME_WITH_DELIMITER, info.getQueueName());
}
@Test
public void testJobHistoryFileNameBackwardsCompatible() throws IOException {
JobID oldJobId = JobID.forName(JOB_ID);