diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fae2bce1d5..022b379770 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -399,6 +399,9 @@ Release 2.4.1 - UNRELEASED HADOOP-10473. TestCallQueueManager should interrupt before counting calls. (szetszwo) + HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors. + (cnauroth) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java index 7e7d855f82..1a571d6659 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.util.Options; @@ -836,21 +837,24 @@ public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); - MapFile.Reader reader = new MapFile.Reader(fs, in, conf); - MapFile.Writer writer = - new MapFile.Writer(conf, fs, out, - reader.getKeyClass().asSubclass(WritableComparable.class), - reader.getValueClass()); + MapFile.Reader reader = null; + MapFile.Writer writer = null; + try { + reader = new MapFile.Reader(fs, in, conf); + writer = + new MapFile.Writer(conf, fs, out, + reader.getKeyClass().asSubclass(WritableComparable.class), + reader.getValueClass()); - WritableComparable key = - ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf); - Writable value = - ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf); + WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass() + .asSubclass(WritableComparable.class), conf); + Writable value = ReflectionUtils.newInstance(reader.getValueClass() + .asSubclass(Writable.class), conf); - while (reader.next(key, value)) // copy all entries - writer.append(key, value); - - writer.close(); + while (reader.next(key, value)) // copy all entries + writer.append(key, value); + } finally { + IOUtils.cleanup(LOG, writer, reader); + } } - } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java index b7b084abcb..b1ea000e9e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java @@ -18,6 +18,8 @@ package org.apache.hadoop.io; +import static org.mockito.Mockito.*; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -31,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionInputStream; @@ -63,39 +66,44 @@ public void testMembershipTest() throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); conf.setInt("io.mapfile.bloom.size", 2048); - BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, - qualifiedDirName.toString(), IntWritable.class, Text.class); - IntWritable key = new IntWritable(); - Text value = new Text(); - for (int i = 0; i < 2000; i += 2) { - key.set(i); - value.set("00" + i); - writer.append(key, value); - } - writer.close(); - - BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, - qualifiedDirName.toString(), conf); - // check false positives rate - int falsePos = 0; - int falseNeg = 0; - for (int i = 0; i < 2000; i++) { - key.set(i); - boolean exists = reader.probablyHasKey(key); - if (i % 2 == 0) { - if (!exists) - falseNeg++; - } else { - if (exists) - falsePos++; + BloomMapFile.Writer writer = null; + BloomMapFile.Reader reader = null; + try { + writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), + IntWritable.class, Text.class); + IntWritable key = new IntWritable(); + Text value = new Text(); + for (int i = 0; i < 2000; i += 2) { + key.set(i); + value.set("00" + i); + writer.append(key, value); } + writer.close(); + + reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); + // check false positives rate + int falsePos = 0; + int falseNeg = 0; + for (int i = 0; i < 2000; i++) { + key.set(i); + boolean exists = reader.probablyHasKey(key); + if (i % 2 == 0) { + if (!exists) + falseNeg++; + } else { + if (exists) + falsePos++; + } + } + reader.close(); + fs.delete(qualifiedDirName, true); + System.out.println("False negatives: " + falseNeg); + assertEquals(0, falseNeg); + System.out.println("False positives: " + falsePos); + assertTrue(falsePos < 2); + } finally { + IOUtils.cleanup(null, writer, reader); } - reader.close(); - fs.delete(qualifiedDirName, true); - System.out.println("False negatives: " + falseNeg); - assertEquals(0, falseNeg); - System.out.println("False positives: " + falsePos); - assertTrue(falsePos < 2); } @SuppressWarnings("deprecation") @@ -103,23 +111,28 @@ private void checkMembershipVaryingSizedKeys(String name, List keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); - BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, - qualifiedDirName.toString(), Text.class, NullWritable.class); - for (Text key : keys) { - writer.append(key, NullWritable.get()); - } - writer.close(); + BloomMapFile.Writer writer = null; + BloomMapFile.Reader reader = null; + try { + writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), + Text.class, NullWritable.class); + for (Text key : keys) { + writer.append(key, NullWritable.get()); + } + writer.close(); - // will check for membership in the opposite order of how keys were inserted - BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, - qualifiedDirName.toString(), conf); - Collections.reverse(keys); - for (Text key : keys) { - assertTrue("False negative for existing key " + key, + // will check for membership in opposite order of how keys were inserted + reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); + Collections.reverse(keys); + for (Text key : keys) { + assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); + } + reader.close(); + fs.delete(qualifiedDirName, true); + } finally { + IOUtils.cleanup(null, writer, reader); } - reader.close(); - fs.delete(qualifiedDirName, true); } public void testMembershipVaryingSizedKeysTest1() throws Exception { @@ -140,15 +153,19 @@ public void testMembershipVaryingSizedKeysTest2() throws Exception { * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { + BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); - BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE, + writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); - BloomMapFile.delete(fs, "." + TEST_FILE); + writer.close(); + BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); + } finally { + IOUtils.cleanup(null, writer); } } @@ -157,24 +174,26 @@ public void testDeleteFile() { * IOException */ public void testIOExceptionInWriterConstructor() { - Path dirNameSpy = org.mockito.Mockito.spy(TEST_FILE); + Path dirNameSpy = spy(TEST_FILE); + BloomMapFile.Reader reader = null; + BloomMapFile.Writer writer = null; try { - BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE, + writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); - org.mockito.Mockito.when(dirNameSpy.getFileSystem(conf)).thenThrow( - new IOException()); - BloomMapFile.Reader reader = new BloomMapFile.Reader(dirNameSpy, conf, + when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); + reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); - reader.close(); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -183,8 +202,10 @@ public void testIOExceptionInWriterConstructor() { */ public void testGetBloomMapFile() { int SIZE = 10; + BloomMapFile.Reader reader = null; + BloomMapFile.Writer writer = null; try { - BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE, + writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); @@ -193,7 +214,7 @@ public void testGetBloomMapFile() { } writer.close(); - BloomMapFile.Reader reader = new BloomMapFile.Reader(TEST_FILE, conf, + reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { @@ -203,9 +224,10 @@ public void testGetBloomMapFile() { assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); - reader.close(); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -214,36 +236,46 @@ public void testGetBloomMapFile() { */ @SuppressWarnings("deprecation") public void testBloomMapFileConstructors() { + BloomMapFile.Writer writer = null; try { FileSystem ts = FileSystem.get(conf); String testFileName = TEST_FILE.toString(); - BloomMapFile.Writer writer1 = new BloomMapFile.Writer(conf, ts, + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultCodec, defaultProgress); - assertNotNull("testBloomMapFileConstructors error !!!", writer1); - BloomMapFile.Writer writer2 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); - assertNotNull("testBloomMapFileConstructors error !!!", writer2); - BloomMapFile.Writer writer3 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); - assertNotNull("testBloomMapFileConstructors error !!!", writer3); - BloomMapFile.Writer writer4 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); - assertNotNull("testBloomMapFileConstructors error !!!", writer4); - BloomMapFile.Writer writer5 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); - assertNotNull("testBloomMapFileConstructors error !!!", writer5); - BloomMapFile.Writer writer6 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); - assertNotNull("testBloomMapFileConstructors error !!!", writer6); - BloomMapFile.Writer writer7 = new BloomMapFile.Writer(conf, ts, + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); + writer = new BloomMapFile.Writer(conf, ts, testFileName, WritableComparator.get(Text.class), Text.class); - assertNotNull("testBloomMapFileConstructors error !!!", writer7); + assertNotNull("testBloomMapFileConstructors error !!!", writer); + writer.close(); } catch (Exception ex) { fail("testBloomMapFileConstructors error !!!"); + } finally { + IOUtils.cleanup(null, writer); } } @@ -272,13 +304,13 @@ public Class getCompressorType() { @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - return null; + return mock(CompressionOutputStream.class); } @Override public CompressionOutputStream createOutputStream(OutputStream out) throws IOException { - return null; + return mock(CompressionOutputStream.class); } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java index f6eff37fd6..34aabb1656 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionInputStream; @@ -68,13 +69,13 @@ public void progress() { @Override public CompressionOutputStream createOutputStream(OutputStream out) throws IOException { - return null; + return mock(CompressionOutputStream.class); } @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - return null; + return mock(CompressionOutputStream.class); } @Override @@ -138,46 +139,52 @@ private MapFile.Reader createReader(String fileName, @Test public void testGetClosestOnCurrentApi() throws Exception { final String TEST_PREFIX = "testGetClosestOnCurrentApi.mapfile"; - MapFile.Writer writer = createWriter(TEST_PREFIX, Text.class, Text.class); - int FIRST_KEY = 1; - // Test keys: 11,21,31,...,91 - for (int i = FIRST_KEY; i < 100; i += 10) { - Text t = new Text(Integer.toString(i)); - writer.append(t, t); + MapFile.Writer writer = null; + MapFile.Reader reader = null; + try { + writer = createWriter(TEST_PREFIX, Text.class, Text.class); + int FIRST_KEY = 1; + // Test keys: 11,21,31,...,91 + for (int i = FIRST_KEY; i < 100; i += 10) { + Text t = new Text(Integer.toString(i)); + writer.append(t, t); + } + writer.close(); + + reader = createReader(TEST_PREFIX, Text.class); + Text key = new Text("55"); + Text value = new Text(); + + // Test get closest with step forward + Text closest = (Text) reader.getClosest(key, value); + assertEquals(new Text("61"), closest); + + // Test get closest with step back + closest = (Text) reader.getClosest(key, value, true); + assertEquals(new Text("51"), closest); + + // Test get closest when we pass explicit key + final Text explicitKey = new Text("21"); + closest = (Text) reader.getClosest(explicitKey, value); + assertEquals(new Text("21"), explicitKey); + + // Test what happens at boundaries. Assert if searching a key that is + // less than first key in the mapfile, that the first key is returned. + key = new Text("00"); + closest = (Text) reader.getClosest(key, value); + assertEquals(FIRST_KEY, Integer.parseInt(closest.toString())); + + // Assert that null is returned if key is > last entry in mapfile. + key = new Text("92"); + closest = (Text) reader.getClosest(key, value); + assertNull("Not null key in testGetClosestWithNewCode", closest); + + // If we were looking for the key before, we should get the last key + closest = (Text) reader.getClosest(key, value, true); + assertEquals(new Text("91"), closest); + } finally { + IOUtils.cleanup(null, writer, reader); } - writer.close(); - - MapFile.Reader reader = createReader(TEST_PREFIX, Text.class); - Text key = new Text("55"); - Text value = new Text(); - - // Test get closest with step forward - Text closest = (Text) reader.getClosest(key, value); - assertEquals(new Text("61"), closest); - - // Test get closest with step back - closest = (Text) reader.getClosest(key, value, true); - assertEquals(new Text("51"), closest); - - // Test get closest when we pass explicit key - final Text explicitKey = new Text("21"); - closest = (Text) reader.getClosest(explicitKey, value); - assertEquals(new Text("21"), explicitKey); - - // Test what happens at boundaries. Assert if searching a key that is - // less than first key in the mapfile, that the first key is returned. - key = new Text("00"); - closest = (Text) reader.getClosest(key, value); - assertEquals(FIRST_KEY, Integer.parseInt(closest.toString())); - - // Assert that null is returned if key is > last entry in mapfile. - key = new Text("92"); - closest = (Text) reader.getClosest(key, value); - assertNull("Not null key in testGetClosestWithNewCode", closest); - - // If we were looking for the key before, we should get the last key - closest = (Text) reader.getClosest(key, value, true); - assertEquals(new Text("91"), closest); } /** @@ -187,16 +194,21 @@ public void testGetClosestOnCurrentApi() throws Exception { public void testMidKeyOnCurrentApi() throws Exception { // Write a mapfile of simple data: keys are final String TEST_PREFIX = "testMidKeyOnCurrentApi.mapfile"; - MapFile.Writer writer = createWriter(TEST_PREFIX, IntWritable.class, - IntWritable.class); - // 0,1,....9 - int SIZE = 10; - for (int i = 0; i < SIZE; i++) - writer.append(new IntWritable(i), new IntWritable(i)); - writer.close(); + MapFile.Writer writer = null; + MapFile.Reader reader = null; + try { + writer = createWriter(TEST_PREFIX, IntWritable.class, IntWritable.class); + // 0,1,....9 + int SIZE = 10; + for (int i = 0; i < SIZE; i++) + writer.append(new IntWritable(i), new IntWritable(i)); + writer.close(); - MapFile.Reader reader = createReader(TEST_PREFIX, IntWritable.class); - assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey()); + reader = createReader(TEST_PREFIX, IntWritable.class); + assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey()); + } finally { + IOUtils.cleanup(null, writer, reader); + } } /** @@ -206,16 +218,18 @@ public void testMidKeyOnCurrentApi() throws Exception { public void testRename() { final String NEW_FILE_NAME = "test-new.mapfile"; final String OLD_FILE_NAME = "test-old.mapfile"; + MapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); - MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class, - IntWritable.class); + writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class); writer.close(); MapFile.rename(fs, new Path(TEST_DIR, OLD_FILE_NAME).toString(), new Path(TEST_DIR, NEW_FILE_NAME).toString()); MapFile.delete(fs, new Path(TEST_DIR, NEW_FILE_NAME).toString()); } catch (IOException ex) { fail("testRename error " + ex); + } finally { + IOUtils.cleanup(null, writer); } } @@ -228,12 +242,12 @@ public void testRenameWithException() { final String ERROR_MESSAGE = "Can't rename file"; final String NEW_FILE_NAME = "test-new.mapfile"; final String OLD_FILE_NAME = "test-old.mapfile"; + MapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); FileSystem spyFs = spy(fs); - MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class, - IntWritable.class); + writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class); writer.close(); Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME); @@ -246,6 +260,8 @@ public void testRenameWithException() { } catch (IOException ex) { assertEquals("testRenameWithException invalid IOExceptionMessage !!!", ex.getMessage(), ERROR_MESSAGE); + } finally { + IOUtils.cleanup(null, writer); } } @@ -254,12 +270,12 @@ public void testRenameWithFalse() { final String ERROR_MESSAGE = "Could not rename"; final String NEW_FILE_NAME = "test-new.mapfile"; final String OLD_FILE_NAME = "test-old.mapfile"; + MapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); FileSystem spyFs = spy(fs); - MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class, - IntWritable.class); + writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class); writer.close(); Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME); @@ -271,6 +287,8 @@ public void testRenameWithFalse() { } catch (IOException ex) { assertTrue("testRenameWithFalse invalid IOExceptionMessage error !!!", ex .getMessage().startsWith(ERROR_MESSAGE)); + } finally { + IOUtils.cleanup(null, writer); } } @@ -297,11 +315,7 @@ public void testWriteWithFailDirCreation() { assertTrue("testWriteWithFailDirCreation ex error !!!", ex.getMessage() .startsWith(ERROR_MESSAGE)); } finally { - if (writer != null) - try { - writer.close(); - } catch (IOException e) { - } + IOUtils.cleanup(null, writer); } } @@ -312,20 +326,24 @@ public void testWriteWithFailDirCreation() { public void testOnFinalKey() { final String TEST_METHOD_KEY = "testOnFinalKey.mapfile"; int SIZE = 10; + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { - MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class, - IntWritable.class); + writer = createWriter(TEST_METHOD_KEY, IntWritable.class, + IntWritable.class); for (int i = 0; i < SIZE; i++) writer.append(new IntWritable(i), new IntWritable(i)); writer.close(); - MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class); + reader = createReader(TEST_METHOD_KEY, IntWritable.class); IntWritable expectedKey = new IntWritable(0); reader.finalKey(expectedKey); assertEquals("testOnFinalKey not same !!!", expectedKey, new IntWritable( 9)); } catch (IOException ex) { fail("testOnFinalKey error !!!"); + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -338,7 +356,8 @@ public void testKeyValueClasses() { Class> keyClass = IntWritable.class; Class valueClass = Text.class; try { - createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class); + createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class) + .close(); assertNotNull("writer key class null error !!!", MapFile.Writer.keyClass(keyClass)); assertNotNull("writer value class null error !!!", @@ -354,19 +373,22 @@ public void testKeyValueClasses() { @Test public void testReaderGetClosest() throws Exception { final String TEST_METHOD_KEY = "testReaderWithWrongKeyClass.mapfile"; + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { - MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class, - Text.class); + writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class); for (int i = 0; i < 10; i++) writer.append(new IntWritable(i), new Text("value" + i)); writer.close(); - MapFile.Reader reader = createReader(TEST_METHOD_KEY, Text.class); + reader = createReader(TEST_METHOD_KEY, Text.class); reader.getClosest(new Text("2"), new Text("")); fail("no excepted exception in testReaderWithWrongKeyClass !!!"); } catch (IOException ex) { /* Should be thrown to pass the test */ + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -376,13 +398,15 @@ public void testReaderGetClosest() throws Exception { @Test public void testReaderWithWrongValueClass() { final String TEST_METHOD_KEY = "testReaderWithWrongValueClass.mapfile"; + MapFile.Writer writer = null; try { - MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class, - Text.class); + writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class); writer.append(new IntWritable(0), new IntWritable(0)); fail("no excepted exception in testReaderWithWrongKeyClass !!!"); } catch (IOException ex) { /* Should be thrown to pass the test */ + } finally { + IOUtils.cleanup(null, writer); } } @@ -394,15 +418,16 @@ public void testReaderKeyIteration() { final String TEST_METHOD_KEY = "testReaderKeyIteration.mapfile"; int SIZE = 10; int ITERATIONS = 5; + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { - MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class, - Text.class); + writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class); int start = 0; for (int i = 0; i < SIZE; i++) writer.append(new IntWritable(i), new Text("Value:" + i)); writer.close(); - MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class); + reader = createReader(TEST_METHOD_KEY, IntWritable.class); // test iteration Writable startValue = new Text("Value:" + start); int i = 0; @@ -421,6 +446,8 @@ public void testReaderKeyIteration() { reader.seek(new IntWritable(SIZE * 2))); } catch (IOException ex) { fail("reader seek error !!!"); + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -431,11 +458,11 @@ public void testReaderKeyIteration() { public void testFix() { final String INDEX_LESS_MAP_FILE = "testFix.mapfile"; int PAIR_SIZE = 20; + MapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); Path dir = new Path(TEST_DIR, INDEX_LESS_MAP_FILE); - MapFile.Writer writer = createWriter(INDEX_LESS_MAP_FILE, - IntWritable.class, Text.class); + writer = createWriter(INDEX_LESS_MAP_FILE, IntWritable.class, Text.class); for (int i = 0; i < PAIR_SIZE; i++) writer.append(new IntWritable(0), new Text("value")); writer.close(); @@ -450,6 +477,8 @@ public void testFix() { MapFile.fix(fs, dir, IntWritable.class, Text.class, true, conf) == PAIR_SIZE); } catch (Exception ex) { fail("testFix error !!!"); + } finally { + IOUtils.cleanup(null, writer); } } /** @@ -459,38 +488,46 @@ public void testFix() { @SuppressWarnings("deprecation") public void testDeprecatedConstructors() { String path = new Path(TEST_DIR, "writes.mapfile").toString(); + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { FileSystem fs = FileSystem.getLocal(conf); - MapFile.Writer writer = new MapFile.Writer(conf, fs, path, + writer = new MapFile.Writer(conf, fs, path, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull(writer); + writer.close(); writer = new MapFile.Writer(conf, fs, path, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgressable); assertNotNull(writer); + writer.close(); writer = new MapFile.Writer(conf, fs, path, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgressable); assertNotNull(writer); + writer.close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.get(Text.class), Text.class); assertNotNull(writer); + writer.close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.get(Text.class), Text.class, SequenceFile.CompressionType.RECORD); assertNotNull(writer); + writer.close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.get(Text.class), Text.class, CompressionType.RECORD, defaultProgressable); assertNotNull(writer); writer.close(); - MapFile.Reader reader = new MapFile.Reader(fs, path, + reader = new MapFile.Reader(fs, path, WritableComparator.get(IntWritable.class), conf); assertNotNull(reader); assertNotNull("reader key is null !!!", reader.getKeyClass()); assertNotNull("reader value in null", reader.getValueClass()); - } catch (IOException e) { fail(e.getMessage()); + } finally { + IOUtils.cleanup(null, writer, reader); } } @@ -509,11 +546,7 @@ public void testKeyLessWriterCreation() { } catch (Exception e) { fail("fail in testKeyLessWriterCreation. Other ex !!!"); } finally { - if (writer != null) - try { - writer.close(); - } catch (IOException e) { - } + IOUtils.cleanup(null, writer); } } /** @@ -542,11 +575,7 @@ public void testPathExplosionWriterCreation() { } catch (Exception e) { fail("fail in testPathExplosionWriterCreation. Other ex !!!"); } finally { - if (writer != null) - try { - writer.close(); - } catch (IOException e) { - } + IOUtils.cleanup(null, writer); } } @@ -555,9 +584,9 @@ public void testPathExplosionWriterCreation() { */ @Test public void testDescOrderWithThrowExceptionWriterAppend() { + MapFile.Writer writer = null; try { - MapFile.Writer writer = createWriter(".mapfile", IntWritable.class, - Text.class); + writer = createWriter(".mapfile", IntWritable.class, Text.class); writer.append(new IntWritable(2), new Text("value: " + 1)); writer.append(new IntWritable(2), new Text("value: " + 2)); writer.append(new IntWritable(2), new Text("value: " + 4)); @@ -566,6 +595,8 @@ public void testDescOrderWithThrowExceptionWriterAppend() { } catch (IOException ex) { } catch (Exception e) { fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!"); + } finally { + IOUtils.cleanup(null, writer); } } @@ -575,15 +606,17 @@ public void testMainMethodMapFile() { String inFile = "mainMethodMapFile.mapfile"; String outFile = "mainMethodMapFile.mapfile"; String[] args = { path, outFile }; + MapFile.Writer writer = null; try { - MapFile.Writer writer = createWriter(inFile, IntWritable.class, - Text.class); + writer = createWriter(inFile, IntWritable.class, Text.class); writer.append(new IntWritable(1), new Text("test_text1")); writer.append(new IntWritable(2), new Text("test_text2")); writer.close(); MapFile.main(args); } catch (Exception ex) { fail("testMainMethodMapFile error !!!"); + } finally { + IOUtils.cleanup(null, writer); } } @@ -601,56 +634,58 @@ public void testGetClosest() throws Exception { Path qualifiedDirName = fs.makeQualified(dirName); // Make an index entry for every third insertion. MapFile.Writer.setIndexInterval(conf, 3); - MapFile.Writer writer = new MapFile.Writer(conf, fs, - qualifiedDirName.toString(), Text.class, Text.class); - // Assert that the index interval is 1 - assertEquals(3, writer.getIndexInterval()); - // Add entries up to 100 in intervals of ten. - final int FIRST_KEY = 10; - for (int i = FIRST_KEY; i < 100; i += 10) { - String iStr = Integer.toString(i); - Text t = new Text("00".substring(iStr.length()) + iStr); - writer.append(t, t); - } - writer.close(); - // Now do getClosest on created mapfile. - MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf); + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { - Text key = new Text("55"); - Text value = new Text(); - Text closest = (Text) reader.getClosest(key, value); - // Assert that closest after 55 is 60 - assertEquals(new Text("60"), closest); - // Get closest that falls before the passed key: 50 - closest = (Text) reader.getClosest(key, value, true); - assertEquals(new Text("50"), closest); - // Test get closest when we pass explicit key - final Text TWENTY = new Text("20"); - closest = (Text) reader.getClosest(TWENTY, value); - assertEquals(TWENTY, closest); - closest = (Text) reader.getClosest(TWENTY, value, true); - assertEquals(TWENTY, closest); - // Test what happens at boundaries. Assert if searching a key that is - // less than first key in the mapfile, that the first key is returned. - key = new Text("00"); - closest = (Text) reader.getClosest(key, value); - assertEquals(FIRST_KEY, Integer.parseInt(closest.toString())); + writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), + Text.class, Text.class); + // Assert that the index interval is 1 + assertEquals(3, writer.getIndexInterval()); + // Add entries up to 100 in intervals of ten. + final int FIRST_KEY = 10; + for (int i = FIRST_KEY; i < 100; i += 10) { + String iStr = Integer.toString(i); + Text t = new Text("00".substring(iStr.length()) + iStr); + writer.append(t, t); + } + writer.close(); + // Now do getClosest on created mapfile. + reader = new MapFile.Reader(qualifiedDirName, conf); + Text key = new Text("55"); + Text value = new Text(); + Text closest = (Text) reader.getClosest(key, value); + // Assert that closest after 55 is 60 + assertEquals(new Text("60"), closest); + // Get closest that falls before the passed key: 50 + closest = (Text) reader.getClosest(key, value, true); + assertEquals(new Text("50"), closest); + // Test get closest when we pass explicit key + final Text TWENTY = new Text("20"); + closest = (Text) reader.getClosest(TWENTY, value); + assertEquals(TWENTY, closest); + closest = (Text) reader.getClosest(TWENTY, value, true); + assertEquals(TWENTY, closest); + // Test what happens at boundaries. Assert if searching a key that is + // less than first key in the mapfile, that the first key is returned. + key = new Text("00"); + closest = (Text) reader.getClosest(key, value); + assertEquals(FIRST_KEY, Integer.parseInt(closest.toString())); - // If we're looking for the first key before, and we pass in a key before - // the first key in the file, we should get null - closest = (Text) reader.getClosest(key, value, true); - assertNull(closest); + // If we're looking for the first key before, and we pass in a key before + // the first key in the file, we should get null + closest = (Text) reader.getClosest(key, value, true); + assertNull(closest); - // Assert that null is returned if key is > last entry in mapfile. - key = new Text("99"); - closest = (Text) reader.getClosest(key, value); - assertNull(closest); + // Assert that null is returned if key is > last entry in mapfile. + key = new Text("99"); + closest = (Text) reader.getClosest(key, value); + assertNull(closest); - // If we were looking for the key before, we should get the last key - closest = (Text) reader.getClosest(key, value, true); - assertEquals(new Text("90"), closest); + // If we were looking for the key before, we should get the last key + closest = (Text) reader.getClosest(key, value, true); + assertEquals(new Text("90"), closest); } finally { - reader.close(); + IOUtils.cleanup(null, writer, reader); } } @@ -662,16 +697,18 @@ public void testMidKey() throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); - MapFile.Writer writer = new MapFile.Writer(conf, fs, - qualifiedDirName.toString(), IntWritable.class, IntWritable.class); - writer.append(new IntWritable(1), new IntWritable(1)); - writer.close(); - // Now do getClosest on created mapfile. - MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf); + MapFile.Writer writer = null; + MapFile.Reader reader = null; try { + writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), + IntWritable.class, IntWritable.class); + writer.append(new IntWritable(1), new IntWritable(1)); + writer.close(); + // Now do getClosest on created mapfile. + reader = new MapFile.Reader(qualifiedDirName, conf); assertEquals(new IntWritable(1), reader.midKey()); } finally { - reader.close(); + IOUtils.cleanup(null, writer, reader); } }