big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment (Mariappan Asokan via jeagles)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1542733 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-11-17 14:29:33 +00:00
parent a02e60fefc
commit fdad179980
3 changed files with 31 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -220,6 +220,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5616. MR Client-AppMaster RPC max retries on socket timeout is too MAPREDUCE-5616. MR Client-AppMaster RPC max retries on socket timeout is too
high. (cnauroth) high. (cnauroth)
MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment
(Mariappan Asokan via jeagles)
Release 2.2.1 - UNRELEASED Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
View File

@ -99,8 +99,7 @@ public void testNoRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
// Set the fixed length record length config property // Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); JobConf job = new JobConf(defaultConf);
JobConf job = new JobConf(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.configure(job); format.configure(job);
@ -127,8 +126,7 @@ public void testZeroRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
// Set the fixed length record length config property // Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); JobConf job = new JobConf(defaultConf);
JobConf job = new JobConf(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job, 0); format.setRecordLength(job, 0);
@ -156,8 +154,7 @@ public void testNegativeRecordLength() throws IOException {
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
// Set the fixed length record length config property // Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); JobConf job = new JobConf(defaultConf);
JobConf job = new JobConf(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job, -10); format.setRecordLength(job, -10);
@ -206,8 +203,8 @@ public void testGzipWithTwoInputs() throws IOException {
writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
"ten nine eightsevensix five four threetwo one "); "ten nine eightsevensix five four threetwo one ");
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
JobConf job = new JobConf(defaultConf); JobConf job = new JobConf(defaultConf);
format.setRecordLength(job, 5);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
ReflectionUtils.setConf(gzip, job); ReflectionUtils.setConf(gzip, job);
format.configure(job); format.configure(job);
@ -290,9 +287,9 @@ private void runRandomTests(CompressionCodec codec) throws IOException {
ArrayList<String> recordList ArrayList<String> recordList
= createFile(file, codec, recordLength, totalRecords); = createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file)); assertTrue(localFs.exists(file));
// Set the fixed length record length config property // Create the job and set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); JobConf job = new JobConf(defaultConf);
FixedLengthInputFormat.setRecordLength(testConf, recordLength); FixedLengthInputFormat.setRecordLength(job, recordLength);
int numSplits = 1; int numSplits = 1;
// Arbitrarily set number of splits. // Arbitrarily set number of splits.
@ -313,8 +310,7 @@ private void runRandomTests(CompressionCodec codec) throws IOException {
LOG.info("Number of splits set to: " + numSplits); LOG.info("Number of splits set to: " + numSplits);
} }
// Create the job, and setup the input path // Setup the input path
JobConf job = new JobConf(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes // Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
@ -390,8 +386,8 @@ private void runPartialRecordTest(CompressionCodec codec) throws IOException {
writeFile(localFs, new Path(workDir, fileName.toString()), codec, writeFile(localFs, new Path(workDir, fileName.toString()), codec,
"one two threefour five six seveneightnine ten"); "one two threefour five six seveneightnine ten");
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
JobConf job = new JobConf(defaultConf); JobConf job = new JobConf(defaultConf);
format.setRecordLength(job, 5);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
if (codec != null) { if (codec != null) {
ReflectionUtils.setConf(codec, job); ReflectionUtils.setConf(codec, job);

40
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
View File

@ -104,9 +104,8 @@ public void testNoRecordLength() throws Exception {
localFs.delete(workDir, true); localFs.delete(workDir, true);
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
// Set the fixed length record length config property // Create the job and do not set fixed record length
Configuration testConf = new Configuration(defaultConf); Job job = Job.getInstance(defaultConf);
Job job = Job.getInstance(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
List<InputSplit> splits = format.getSplits(job); List<InputSplit> splits = format.getSplits(job);
@ -139,11 +138,10 @@ public void testZeroRecordLength() throws Exception {
localFs.delete(workDir, true); localFs.delete(workDir, true);
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
Job job = Job.getInstance(defaultConf);
// Set the fixed length record length config property // Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(testConf, 0); format.setRecordLength(job.getConfiguration(), 0);
Job job = Job.getInstance(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job); List<InputSplit> splits = format.getSplits(job);
boolean exceptionThrown = false; boolean exceptionThrown = false;
@ -177,10 +175,9 @@ public void testNegativeRecordLength() throws Exception {
Path file = new Path(workDir, new String("testFormat.txt")); Path file = new Path(workDir, new String("testFormat.txt"));
createFile(file, null, 10, 10); createFile(file, null, 10, 10);
// Set the fixed length record length config property // Set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(testConf, -10); format.setRecordLength(job.getConfiguration(), -10);
Job job = Job.getInstance(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job); List<InputSplit> splits = format.getSplits(job);
boolean exceptionThrown = false; boolean exceptionThrown = false;
@ -233,10 +230,10 @@ public void testGzipWithTwoInputs() throws Exception {
"one two threefour five six seveneightnine ten "); "one two threefour five six seveneightnine ten ");
writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip,
"ten nine eightsevensix five four threetwo one "); "ten nine eightsevensix five four threetwo one ");
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5);
ReflectionUtils.setConf(gzip, defaultConf);
Job job = Job.getInstance(defaultConf); Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(job.getConfiguration(), 5);
ReflectionUtils.setConf(gzip, job.getConfiguration());
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job); List<InputSplit> splits = format.getSplits(job);
assertEquals("compressed splits == 2", 2, splits.size()); assertEquals("compressed splits == 2", 2, splits.size());
@ -317,9 +314,10 @@ private void runRandomTests(CompressionCodec codec) throws Exception {
ArrayList<String> recordList = ArrayList<String> recordList =
createFile(file, codec, recordLength, totalRecords); createFile(file, codec, recordLength, totalRecords);
assertTrue(localFs.exists(file)); assertTrue(localFs.exists(file));
// Set the fixed length record length config property // Create the job and set the fixed length record length config property
Configuration testConf = new Configuration(defaultConf); Job job = Job.getInstance(defaultConf);
FixedLengthInputFormat.setRecordLength(testConf, recordLength); FixedLengthInputFormat.setRecordLength(job.getConfiguration(),
recordLength);
int numSplits = 1; int numSplits = 1;
// Arbitrarily set number of splits. // Arbitrarily set number of splits.
@ -339,11 +337,11 @@ private void runRandomTests(CompressionCodec codec) throws Exception {
} }
LOG.info("Number of splits set to: " + numSplits); LOG.info("Number of splits set to: " + numSplits);
} }
testConf.setLong("mapreduce.input.fileinputformat.split.maxsize", job.getConfiguration().setLong(
"mapreduce.input.fileinputformat.split.maxsize",
(long)(fileSize/numSplits)); (long)(fileSize/numSplits));
// Create the job, and setup the input path // setup the input path
Job job = Job.getInstance(testConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
// Try splitting the file in a variety of sizes // Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
@ -429,18 +427,18 @@ private static List<String> readSplit(FixedLengthInputFormat format,
private void runPartialRecordTest(CompressionCodec codec) throws Exception { private void runPartialRecordTest(CompressionCodec codec) throws Exception {
localFs.delete(workDir, true); localFs.delete(workDir, true);
Job job = Job.getInstance(defaultConf);
// Create a file with fixed length records with 5 byte long // Create a file with fixed length records with 5 byte long
// records with a partial record at the end. // records with a partial record at the end.
StringBuilder fileName = new StringBuilder("testFormat.txt"); StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null) { if (codec != null) {
fileName.append(".gz"); fileName.append(".gz");
ReflectionUtils.setConf(codec, defaultConf); ReflectionUtils.setConf(codec, job.getConfiguration());
} }
writeFile(localFs, new Path(workDir, fileName.toString()), codec, writeFile(localFs, new Path(workDir, fileName.toString()), codec,
"one two threefour five six seveneightnine ten"); "one two threefour five six seveneightnine ten");
FixedLengthInputFormat format = new FixedLengthInputFormat(); FixedLengthInputFormat format = new FixedLengthInputFormat();
format.setRecordLength(defaultConf, 5); format.setRecordLength(job.getConfiguration(), 5);
Job job = Job.getInstance(defaultConf);
FileInputFormat.setInputPaths(job, workDir); FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job); List<InputSplit> splits = format.getSplits(job);
if (codec != null) { if (codec != null) {