big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment (Mariappan Asokan via jeagles)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1542733 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-11-17 14:29:33 +00:00
parent a02e60fefc
commit fdad179980
3 changed files with 31 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -220,6 +220,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5616. MR Client-AppMaster RPC max retries on socket timeout is too
high. (cnauroth)
MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment
(Mariappan Asokan via jeagles)
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
View File

@ -99,8 +99,7 @@ public void testNoRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
JobConf job = new JobConf(testConf);
JobConf job = new JobConf(defaultConf);
FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job);
@ -127,8 +126,7 @@ public void testZeroRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
JobConf job = new JobConf(testConf);
JobConf job = new JobConf(defaultConf);
FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job, 0);
@ -156,8 +154,7 @@ public void testNegativeRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
JobConf job = new JobConf(testConf);
JobConf job = new JobConf(defaultConf);
FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job, -10);
@ -206,8 +203,8 @@ public void testGzipWithTwoInputs() throws IOException {
writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
"ten nine eightsevensix five four threetwo one ");
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
JobConf job = new JobConf(defaultConf);
format.setRecordLength(job, 5);
FileInputFormat.setInputPaths(job, workDir);
ReflectionUtils.setConf(gzip, job);
format.configure(job);
@ -290,9 +287,9 @@ private void runRandomTests(CompressionCodec codec) throws IOException {
ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
FixedLengthInputFormat.setRecordLength(testConf, recordLength);
// Create the job and set the fixed length record length config property
JobConf job = new JobConf(defaultConf);
FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
@ -313,8 +310,7 @@ private void runRandomTests(CompressionCodec codec) throws IOException {
LOG.info("Number of splits set to: " + numSplits);
}
// Create the job, and setup the input path
JobConf job = new JobConf(testConf);
// Setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
@ -390,8 +386,8 @@ private void runPartialRecordTest(CompressionCodec codec) throws IOException {
writeFile(localFs, new Path(workDir, fileName.toString()), codec,
"one two threefour five six seveneightnine ten");
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
JobConf job = new JobConf(defaultConf);
format.setRecordLength(job, 5);
FileInputFormat.setInputPaths(job, workDir);
if (codec != null) {
ReflectionUtils.setConf(codec, job);

40
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
View File

@ -104,9 +104,8 @@ public void testNoRecordLength() throws Exception {
localFs.delete(workDir, true);
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
Job job = Job.getInstance(testConf);
// Create the job and do not set fixed record length
Job job = Job.getInstance(defaultConf);
FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat();
List<InputSplit> splits = format.getSplits(job);
@ -139,11 +138,10 @@ public void testZeroRecordLength() throws Exception {
localFs.delete(workDir, true);
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
Job job = Job.getInstance(defaultConf);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(testConf, 0);
Job job = Job.getInstance(testConf);
format.setRecordLength(job.getConfiguration(), 0);
FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job);
boolean exceptionThrown = false;
@ -177,10 +175,9 @@ public void testNegativeRecordLength() throws Exception {
Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10);
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(testConf, -10);
Job job = Job.getInstance(testConf);
format.setRecordLength(job.getConfiguration(), -10);
FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job);
boolean exceptionThrown = false;
@ -233,10 +230,10 @@ public void testGzipWithTwoInputs() throws Exception {
"one two threefour five six seveneightnine ten ");
writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
"ten nine eightsevensix five four threetwo one ");
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
ReflectionUtils.setConf(gzip, defaultConf);
Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job.getConfiguration(), 5);
ReflectionUtils.setConf(gzip, job.getConfiguration());
FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job);
assertEquals("compressed splits == 2", 2, splits.size());
@ -317,9 +314,10 @@ private void runRandomTests(CompressionCodec codec) throws Exception {
ArrayList<String> recordList =
createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file));
// Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
FixedLengthInputFormat.setRecordLength(testConf, recordLength);
// Create the job and set the fixed length record length config property
Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat.setRecordLength(job.getConfiguration(),
recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
@ -339,11 +337,11 @@ private void runRandomTests(CompressionCodec codec) throws Exception {
}
LOG.info("Number of splits set to: " + numSplits);
}
testConf.setLong("mapreduce.input.fileinputformat.split.maxsize",
job.getConfiguration().setLong(
"mapreduce.input.fileinputformat.split.maxsize",
(long)(fileSize/numSplits));
// Create the job, and setup the input path
Job job = Job.getInstance(testConf);
// setup the input path
FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
@ -429,18 +427,18 @@ private static List<String> readSplit(FixedLengthInputFormat format,
private void runPartialRecordTest(CompressionCodec codec) throws Exception {
localFs.delete(workDir, true);
Job job = Job.getInstance(defaultConf);
// Create a file with fixed length records with 5 byte long
// records with a partial record at the end.
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) {
fileName.append(".gz");
ReflectionUtils.setConf(codec, defaultConf);
ReflectionUtils.setConf(codec, job.getConfiguration());
}
writeFile(localFs, new Path(workDir, fileName.toString()), codec,
"one two threefour five six seveneightnine ten");
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
Job job = Job.getInstance(defaultConf);
format.setRecordLength(job.getConfiguration(), 5);
FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job);
if (codec != null) {