From fdad179980f7b5a8222c8462b55b06ab615f64e5 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Sun, 17 Nov 2013 14:29:33 +0000 Subject: [PATCH] MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment (Mariappan Asokan via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1542733 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapred/TestFixedLengthInputFormat.java | 22 +++++----- .../lib/input/TestFixedLengthInputFormat.java | 40 +++++++++---------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8e9568ee51..c066e6240a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -220,6 +220,9 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5616. MR Client-AppMaster RPC max retries on socket timeout is too high. (cnauroth) + MAPREDUCE-5625. TestFixedLengthInputFormat fails in jdk7 environment + (Mariappan Asokan via jeagles) + Release 2.2.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java index 0c614eaf0c..a5f19eeb25 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java @@ -99,8 +99,7 @@ public void testNoRecordLength() throws IOException { Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - JobConf job = new JobConf(testConf); + JobConf job = new JobConf(defaultConf); FileInputFormat.setInputPaths(job, workDir); FixedLengthInputFormat format = new FixedLengthInputFormat(); format.configure(job); @@ -127,8 +126,7 @@ public void testZeroRecordLength() throws IOException { Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - JobConf job = new JobConf(testConf); + JobConf job = new JobConf(defaultConf); FileInputFormat.setInputPaths(job, workDir); FixedLengthInputFormat format = new FixedLengthInputFormat(); format.setRecordLength(job, 0); @@ -156,8 +154,7 @@ public void testNegativeRecordLength() throws IOException { Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - JobConf job = new JobConf(testConf); + JobConf job = new JobConf(defaultConf); FileInputFormat.setInputPaths(job, workDir); FixedLengthInputFormat format = new FixedLengthInputFormat(); format.setRecordLength(job, -10); @@ -206,8 +203,8 @@ public void testGzipWithTwoInputs() throws IOException { writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, "ten nine eightsevensix five four threetwo one "); FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(defaultConf, 5); JobConf job = new JobConf(defaultConf); + format.setRecordLength(job, 5); FileInputFormat.setInputPaths(job, workDir); ReflectionUtils.setConf(gzip, job); format.configure(job); @@ -290,9 +287,9 @@ private void runRandomTests(CompressionCodec codec) throws IOException { ArrayList recordList = createFile(file, codec, recordLength, totalRecords); assertTrue(localFs.exists(file)); - // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - FixedLengthInputFormat.setRecordLength(testConf, recordLength); + // Create the job and set the fixed length record length config property + JobConf job = new JobConf(defaultConf); + FixedLengthInputFormat.setRecordLength(job, recordLength); int numSplits = 1; // Arbitrarily set number of splits. @@ -313,8 +310,7 @@ private void runRandomTests(CompressionCodec codec) throws IOException { LOG.info("Number of splits set to: " + numSplits); } - // Create the job, and setup the input path - JobConf job = new JobConf(testConf); + // Setup the input path FileInputFormat.setInputPaths(job, workDir); // Try splitting the file in a variety of sizes FixedLengthInputFormat format = new FixedLengthInputFormat(); @@ -390,8 +386,8 @@ private void runPartialRecordTest(CompressionCodec codec) throws IOException { writeFile(localFs, new Path(workDir, fileName.toString()), codec, "one two threefour five six seveneightnine ten"); FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(defaultConf, 5); JobConf job = new JobConf(defaultConf); + format.setRecordLength(job, 5); FileInputFormat.setInputPaths(job, workDir); if (codec != null) { ReflectionUtils.setConf(codec, job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java index 0c94a4d34c..f00b1a92ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java @@ -104,9 +104,8 @@ public void testNoRecordLength() throws Exception { localFs.delete(workDir, true); Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); - // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - Job job = Job.getInstance(testConf); + // Create the job and do not set fixed record length + Job job = Job.getInstance(defaultConf); FileInputFormat.setInputPaths(job, workDir); FixedLengthInputFormat format = new FixedLengthInputFormat(); List splits = format.getSplits(job); @@ -139,11 +138,10 @@ public void testZeroRecordLength() throws Exception { localFs.delete(workDir, true); Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); + Job job = Job.getInstance(defaultConf); // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(testConf, 0); - Job job = Job.getInstance(testConf); + format.setRecordLength(job.getConfiguration(), 0); FileInputFormat.setInputPaths(job, workDir); List splits = format.getSplits(job); boolean exceptionThrown = false; @@ -177,10 +175,9 @@ public void testNegativeRecordLength() throws Exception { Path file = new Path(workDir, new String("testFormat.txt")); createFile(file, null, 10, 10); // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); + Job job = Job.getInstance(defaultConf); FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(testConf, -10); - Job job = Job.getInstance(testConf); + format.setRecordLength(job.getConfiguration(), -10); FileInputFormat.setInputPaths(job, workDir); List splits = format.getSplits(job); boolean exceptionThrown = false; @@ -233,10 +230,10 @@ public void testGzipWithTwoInputs() throws Exception { "one two threefour five six seveneightnine ten "); writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, "ten nine eightsevensix five four threetwo one "); - FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(defaultConf, 5); - ReflectionUtils.setConf(gzip, defaultConf); Job job = Job.getInstance(defaultConf); + FixedLengthInputFormat format = new FixedLengthInputFormat(); + format.setRecordLength(job.getConfiguration(), 5); + ReflectionUtils.setConf(gzip, job.getConfiguration()); FileInputFormat.setInputPaths(job, workDir); List splits = format.getSplits(job); assertEquals("compressed splits == 2", 2, splits.size()); @@ -317,9 +314,10 @@ private void runRandomTests(CompressionCodec codec) throws Exception { ArrayList recordList = createFile(file, codec, recordLength, totalRecords); assertTrue(localFs.exists(file)); - // Set the fixed length record length config property - Configuration testConf = new Configuration(defaultConf); - FixedLengthInputFormat.setRecordLength(testConf, recordLength); + // Create the job and set the fixed length record length config property + Job job = Job.getInstance(defaultConf); + FixedLengthInputFormat.setRecordLength(job.getConfiguration(), + recordLength); int numSplits = 1; // Arbitrarily set number of splits. @@ -339,11 +337,11 @@ private void runRandomTests(CompressionCodec codec) throws Exception { } LOG.info("Number of splits set to: " + numSplits); } - testConf.setLong("mapreduce.input.fileinputformat.split.maxsize", + job.getConfiguration().setLong( + "mapreduce.input.fileinputformat.split.maxsize", (long)(fileSize/numSplits)); - // Create the job, and setup the input path - Job job = Job.getInstance(testConf); + // setup the input path FileInputFormat.setInputPaths(job, workDir); // Try splitting the file in a variety of sizes FixedLengthInputFormat format = new FixedLengthInputFormat(); @@ -429,18 +427,18 @@ private static List readSplit(FixedLengthInputFormat format, private void runPartialRecordTest(CompressionCodec codec) throws Exception { localFs.delete(workDir, true); + Job job = Job.getInstance(defaultConf); // Create a file with fixed length records with 5 byte long // records with a partial record at the end. StringBuilder fileName = new StringBuilder("testFormat.txt"); if (codec != null) { fileName.append(".gz"); - ReflectionUtils.setConf(codec, defaultConf); + ReflectionUtils.setConf(codec, job.getConfiguration()); } writeFile(localFs, new Path(workDir, fileName.toString()), codec, "one two threefour five six seveneightnine ten"); FixedLengthInputFormat format = new FixedLengthInputFormat(); - format.setRecordLength(defaultConf, 5); - Job job = Job.getInstance(defaultConf); + format.setRecordLength(job.getConfiguration(), 5); FileInputFormat.setInputPaths(job, workDir); List splits = format.getSplits(job); if (codec != null) {