HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors. Contributed by Chris Nauroth.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1586570 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth
parent
c1127d19c4
commit
91ebf58904
@ -399,6 +399,9 @@ Release 2.4.1 - UNRELEASED
|
||||
HADOOP-10473. TestCallQueueManager should interrupt before counting calls.
|
||||
(szetszwo)
|
||||
|
||||
HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors.
|
||||
(cnauroth)
|
||||
|
||||
Release 2.4.0 - 2014-04-07
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.io.SequenceFile.CompressionType;
|
||||
import org.apache.hadoop.io.compress.CompressionCodec;
|
||||
import org.apache.hadoop.util.Options;
|
||||
@ -836,21 +837,24 @@ public static void main(String[] args) throws Exception {
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
MapFile.Reader reader = new MapFile.Reader(fs, in, conf);
|
||||
MapFile.Writer writer =
|
||||
MapFile.Reader reader = null;
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
reader = new MapFile.Reader(fs, in, conf);
|
||||
writer =
|
||||
new MapFile.Writer(conf, fs, out,
|
||||
reader.getKeyClass().asSubclass(WritableComparable.class),
|
||||
reader.getValueClass());
|
||||
|
||||
WritableComparable key =
|
||||
ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf);
|
||||
Writable value =
|
||||
ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf);
|
||||
WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass()
|
||||
.asSubclass(WritableComparable.class), conf);
|
||||
Writable value = ReflectionUtils.newInstance(reader.getValueClass()
|
||||
.asSubclass(Writable.class), conf);
|
||||
|
||||
while (reader.next(key, value)) // copy all entries
|
||||
writer.append(key, value);
|
||||
|
||||
writer.close();
|
||||
} finally {
|
||||
IOUtils.cleanup(LOG, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -18,6 +18,8 @@
|
||||
|
||||
package org.apache.hadoop.io;
|
||||
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
@ -31,6 +33,7 @@
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.io.SequenceFile.CompressionType;
|
||||
import org.apache.hadoop.io.compress.CompressionCodec;
|
||||
import org.apache.hadoop.io.compress.CompressionInputStream;
|
||||
@ -63,8 +66,11 @@ public void testMembershipTest() throws Exception {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
Path qualifiedDirName = fs.makeQualified(TEST_DIR);
|
||||
conf.setInt("io.mapfile.bloom.size", 2048);
|
||||
BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
|
||||
qualifiedDirName.toString(), IntWritable.class, Text.class);
|
||||
BloomMapFile.Writer writer = null;
|
||||
BloomMapFile.Reader reader = null;
|
||||
try {
|
||||
writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
|
||||
IntWritable.class, Text.class);
|
||||
IntWritable key = new IntWritable();
|
||||
Text value = new Text();
|
||||
for (int i = 0; i < 2000; i += 2) {
|
||||
@ -74,8 +80,7 @@ public void testMembershipTest() throws Exception {
|
||||
}
|
||||
writer.close();
|
||||
|
||||
BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
|
||||
qualifiedDirName.toString(), conf);
|
||||
reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
|
||||
// check false positives rate
|
||||
int falsePos = 0;
|
||||
int falseNeg = 0;
|
||||
@ -96,6 +101,9 @@ public void testMembershipTest() throws Exception {
|
||||
assertEquals(0, falseNeg);
|
||||
System.out.println("False positives: " + falsePos);
|
||||
assertTrue(falsePos < 2);
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@ -103,16 +111,18 @@ private void checkMembershipVaryingSizedKeys(String name, List<Text> keys)
|
||||
throws Exception {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
Path qualifiedDirName = fs.makeQualified(TEST_DIR);
|
||||
BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
|
||||
qualifiedDirName.toString(), Text.class, NullWritable.class);
|
||||
BloomMapFile.Writer writer = null;
|
||||
BloomMapFile.Reader reader = null;
|
||||
try {
|
||||
writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
|
||||
Text.class, NullWritable.class);
|
||||
for (Text key : keys) {
|
||||
writer.append(key, NullWritable.get());
|
||||
}
|
||||
writer.close();
|
||||
|
||||
// will check for membership in the opposite order of how keys were inserted
|
||||
BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
|
||||
qualifiedDirName.toString(), conf);
|
||||
// will check for membership in opposite order of how keys were inserted
|
||||
reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
|
||||
Collections.reverse(keys);
|
||||
for (Text key : keys) {
|
||||
assertTrue("False negative for existing key " + key,
|
||||
@ -120,6 +130,9 @@ private void checkMembershipVaryingSizedKeys(String name, List<Text> keys)
|
||||
}
|
||||
reader.close();
|
||||
fs.delete(qualifiedDirName, true);
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
public void testMembershipVaryingSizedKeysTest1() throws Exception {
|
||||
@ -140,15 +153,19 @@ public void testMembershipVaryingSizedKeysTest2() throws Exception {
|
||||
* test {@code BloomMapFile.delete()} method
|
||||
*/
|
||||
public void testDeleteFile() {
|
||||
BloomMapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
MapFile.Writer.keyClass(IntWritable.class),
|
||||
MapFile.Writer.valueClass(Text.class));
|
||||
assertNotNull("testDeleteFile error !!!", writer);
|
||||
BloomMapFile.delete(fs, "." + TEST_FILE);
|
||||
writer.close();
|
||||
BloomMapFile.delete(fs, TEST_FILE.toString());
|
||||
} catch (Exception ex) {
|
||||
fail("unexpect ex in testDeleteFile !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -157,24 +174,26 @@ public void testDeleteFile() {
|
||||
* IOException
|
||||
*/
|
||||
public void testIOExceptionInWriterConstructor() {
|
||||
Path dirNameSpy = org.mockito.Mockito.spy(TEST_FILE);
|
||||
Path dirNameSpy = spy(TEST_FILE);
|
||||
BloomMapFile.Reader reader = null;
|
||||
BloomMapFile.Writer writer = null;
|
||||
try {
|
||||
BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
MapFile.Writer.keyClass(IntWritable.class),
|
||||
MapFile.Writer.valueClass(Text.class));
|
||||
writer.append(new IntWritable(1), new Text("123124142"));
|
||||
writer.close();
|
||||
|
||||
org.mockito.Mockito.when(dirNameSpy.getFileSystem(conf)).thenThrow(
|
||||
new IOException());
|
||||
BloomMapFile.Reader reader = new BloomMapFile.Reader(dirNameSpy, conf,
|
||||
when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException());
|
||||
reader = new BloomMapFile.Reader(dirNameSpy, conf,
|
||||
MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
|
||||
|
||||
assertNull("testIOExceptionInWriterConstructor error !!!",
|
||||
reader.getBloomFilter());
|
||||
reader.close();
|
||||
} catch (Exception ex) {
|
||||
fail("unexpect ex in testIOExceptionInWriterConstructor !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -183,8 +202,10 @@ public void testIOExceptionInWriterConstructor() {
|
||||
*/
|
||||
public void testGetBloomMapFile() {
|
||||
int SIZE = 10;
|
||||
BloomMapFile.Reader reader = null;
|
||||
BloomMapFile.Writer writer = null;
|
||||
try {
|
||||
BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
writer = new BloomMapFile.Writer(conf, TEST_FILE,
|
||||
MapFile.Writer.keyClass(IntWritable.class),
|
||||
MapFile.Writer.valueClass(Text.class));
|
||||
|
||||
@ -193,7 +214,7 @@ public void testGetBloomMapFile() {
|
||||
}
|
||||
writer.close();
|
||||
|
||||
BloomMapFile.Reader reader = new BloomMapFile.Reader(TEST_FILE, conf,
|
||||
reader = new BloomMapFile.Reader(TEST_FILE, conf,
|
||||
MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
|
||||
|
||||
for (int i = 0; i < SIZE; i++) {
|
||||
@ -203,9 +224,10 @@ public void testGetBloomMapFile() {
|
||||
|
||||
assertNull("testGetBloomMapFile error !!!",
|
||||
reader.get(new IntWritable(SIZE + 5), new Text()));
|
||||
reader.close();
|
||||
} catch (Exception ex) {
|
||||
fail("unexpect ex in testGetBloomMapFile !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,36 +236,46 @@ public void testGetBloomMapFile() {
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testBloomMapFileConstructors() {
|
||||
BloomMapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem ts = FileSystem.get(conf);
|
||||
String testFileName = TEST_FILE.toString();
|
||||
BloomMapFile.Writer writer1 = new BloomMapFile.Writer(conf, ts,
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
|
||||
defaultCodec, defaultProgress);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer1);
|
||||
BloomMapFile.Writer writer2 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
|
||||
defaultProgress);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer2);
|
||||
BloomMapFile.Writer writer3 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.BLOCK);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer3);
|
||||
BloomMapFile.Writer writer4 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
|
||||
defaultCodec, defaultProgress);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer4);
|
||||
BloomMapFile.Writer writer5 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
|
||||
defaultProgress);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer5);
|
||||
BloomMapFile.Writer writer6 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, IntWritable.class, Text.class, CompressionType.RECORD);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer6);
|
||||
BloomMapFile.Writer writer7 = new BloomMapFile.Writer(conf, ts,
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
writer = new BloomMapFile.Writer(conf, ts,
|
||||
testFileName, WritableComparator.get(Text.class), Text.class);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer7);
|
||||
assertNotNull("testBloomMapFileConstructors error !!!", writer);
|
||||
writer.close();
|
||||
} catch (Exception ex) {
|
||||
fail("testBloomMapFileConstructors error !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -272,13 +304,13 @@ public Class<? extends Compressor> getCompressorType() {
|
||||
@Override
|
||||
public CompressionOutputStream createOutputStream(OutputStream out,
|
||||
Compressor compressor) throws IOException {
|
||||
return null;
|
||||
return mock(CompressionOutputStream.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompressionOutputStream createOutputStream(OutputStream out)
|
||||
throws IOException {
|
||||
return null;
|
||||
return mock(CompressionOutputStream.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.io.SequenceFile.CompressionType;
|
||||
import org.apache.hadoop.io.compress.CompressionCodec;
|
||||
import org.apache.hadoop.io.compress.CompressionInputStream;
|
||||
@ -68,13 +69,13 @@ public void progress() {
|
||||
@Override
|
||||
public CompressionOutputStream createOutputStream(OutputStream out)
|
||||
throws IOException {
|
||||
return null;
|
||||
return mock(CompressionOutputStream.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompressionOutputStream createOutputStream(OutputStream out,
|
||||
Compressor compressor) throws IOException {
|
||||
return null;
|
||||
return mock(CompressionOutputStream.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -138,7 +139,10 @@ private MapFile.Reader createReader(String fileName,
|
||||
@Test
|
||||
public void testGetClosestOnCurrentApi() throws Exception {
|
||||
final String TEST_PREFIX = "testGetClosestOnCurrentApi.mapfile";
|
||||
MapFile.Writer writer = createWriter(TEST_PREFIX, Text.class, Text.class);
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
writer = createWriter(TEST_PREFIX, Text.class, Text.class);
|
||||
int FIRST_KEY = 1;
|
||||
// Test keys: 11,21,31,...,91
|
||||
for (int i = FIRST_KEY; i < 100; i += 10) {
|
||||
@ -147,7 +151,7 @@ public void testGetClosestOnCurrentApi() throws Exception {
|
||||
}
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = createReader(TEST_PREFIX, Text.class);
|
||||
reader = createReader(TEST_PREFIX, Text.class);
|
||||
Text key = new Text("55");
|
||||
Text value = new Text();
|
||||
|
||||
@ -178,6 +182,9 @@ public void testGetClosestOnCurrentApi() throws Exception {
|
||||
// If we were looking for the key before, we should get the last key
|
||||
closest = (Text) reader.getClosest(key, value, true);
|
||||
assertEquals(new Text("91"), closest);
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -187,16 +194,21 @@ public void testGetClosestOnCurrentApi() throws Exception {
|
||||
public void testMidKeyOnCurrentApi() throws Exception {
|
||||
// Write a mapfile of simple data: keys are
|
||||
final String TEST_PREFIX = "testMidKeyOnCurrentApi.mapfile";
|
||||
MapFile.Writer writer = createWriter(TEST_PREFIX, IntWritable.class,
|
||||
IntWritable.class);
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
writer = createWriter(TEST_PREFIX, IntWritable.class, IntWritable.class);
|
||||
// 0,1,....9
|
||||
int SIZE = 10;
|
||||
for (int i = 0; i < SIZE; i++)
|
||||
writer.append(new IntWritable(i), new IntWritable(i));
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = createReader(TEST_PREFIX, IntWritable.class);
|
||||
reader = createReader(TEST_PREFIX, IntWritable.class);
|
||||
assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -206,16 +218,18 @@ public void testMidKeyOnCurrentApi() throws Exception {
|
||||
public void testRename() {
|
||||
final String NEW_FILE_NAME = "test-new.mapfile";
|
||||
final String OLD_FILE_NAME = "test-old.mapfile";
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
|
||||
IntWritable.class);
|
||||
writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
|
||||
writer.close();
|
||||
MapFile.rename(fs, new Path(TEST_DIR, OLD_FILE_NAME).toString(),
|
||||
new Path(TEST_DIR, NEW_FILE_NAME).toString());
|
||||
MapFile.delete(fs, new Path(TEST_DIR, NEW_FILE_NAME).toString());
|
||||
} catch (IOException ex) {
|
||||
fail("testRename error " + ex);
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -228,12 +242,12 @@ public void testRenameWithException() {
|
||||
final String ERROR_MESSAGE = "Can't rename file";
|
||||
final String NEW_FILE_NAME = "test-new.mapfile";
|
||||
final String OLD_FILE_NAME = "test-old.mapfile";
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
FileSystem spyFs = spy(fs);
|
||||
|
||||
MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
|
||||
IntWritable.class);
|
||||
writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
|
||||
writer.close();
|
||||
|
||||
Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
|
||||
@ -246,6 +260,8 @@ public void testRenameWithException() {
|
||||
} catch (IOException ex) {
|
||||
assertEquals("testRenameWithException invalid IOExceptionMessage !!!",
|
||||
ex.getMessage(), ERROR_MESSAGE);
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -254,12 +270,12 @@ public void testRenameWithFalse() {
|
||||
final String ERROR_MESSAGE = "Could not rename";
|
||||
final String NEW_FILE_NAME = "test-new.mapfile";
|
||||
final String OLD_FILE_NAME = "test-old.mapfile";
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
FileSystem spyFs = spy(fs);
|
||||
|
||||
MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
|
||||
IntWritable.class);
|
||||
writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
|
||||
writer.close();
|
||||
|
||||
Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
|
||||
@ -271,6 +287,8 @@ public void testRenameWithFalse() {
|
||||
} catch (IOException ex) {
|
||||
assertTrue("testRenameWithFalse invalid IOExceptionMessage error !!!", ex
|
||||
.getMessage().startsWith(ERROR_MESSAGE));
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -297,11 +315,7 @@ public void testWriteWithFailDirCreation() {
|
||||
assertTrue("testWriteWithFailDirCreation ex error !!!", ex.getMessage()
|
||||
.startsWith(ERROR_MESSAGE));
|
||||
} finally {
|
||||
if (writer != null)
|
||||
try {
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -312,20 +326,24 @@ public void testWriteWithFailDirCreation() {
|
||||
public void testOnFinalKey() {
|
||||
final String TEST_METHOD_KEY = "testOnFinalKey.mapfile";
|
||||
int SIZE = 10;
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
|
||||
writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
|
||||
IntWritable.class);
|
||||
for (int i = 0; i < SIZE; i++)
|
||||
writer.append(new IntWritable(i), new IntWritable(i));
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
|
||||
reader = createReader(TEST_METHOD_KEY, IntWritable.class);
|
||||
IntWritable expectedKey = new IntWritable(0);
|
||||
reader.finalKey(expectedKey);
|
||||
assertEquals("testOnFinalKey not same !!!", expectedKey, new IntWritable(
|
||||
9));
|
||||
} catch (IOException ex) {
|
||||
fail("testOnFinalKey error !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -338,7 +356,8 @@ public void testKeyValueClasses() {
|
||||
Class<? extends WritableComparable<?>> keyClass = IntWritable.class;
|
||||
Class<?> valueClass = Text.class;
|
||||
try {
|
||||
createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class);
|
||||
createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class)
|
||||
.close();
|
||||
assertNotNull("writer key class null error !!!",
|
||||
MapFile.Writer.keyClass(keyClass));
|
||||
assertNotNull("writer value class null error !!!",
|
||||
@ -354,19 +373,22 @@ public void testKeyValueClasses() {
|
||||
@Test
|
||||
public void testReaderGetClosest() throws Exception {
|
||||
final String TEST_METHOD_KEY = "testReaderWithWrongKeyClass.mapfile";
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
|
||||
Text.class);
|
||||
writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
writer.append(new IntWritable(i), new Text("value" + i));
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = createReader(TEST_METHOD_KEY, Text.class);
|
||||
reader = createReader(TEST_METHOD_KEY, Text.class);
|
||||
reader.getClosest(new Text("2"), new Text(""));
|
||||
fail("no excepted exception in testReaderWithWrongKeyClass !!!");
|
||||
} catch (IOException ex) {
|
||||
/* Should be thrown to pass the test */
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -376,13 +398,15 @@ public void testReaderGetClosest() throws Exception {
|
||||
@Test
|
||||
public void testReaderWithWrongValueClass() {
|
||||
final String TEST_METHOD_KEY = "testReaderWithWrongValueClass.mapfile";
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
|
||||
Text.class);
|
||||
writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
|
||||
writer.append(new IntWritable(0), new IntWritable(0));
|
||||
fail("no excepted exception in testReaderWithWrongKeyClass !!!");
|
||||
} catch (IOException ex) {
|
||||
/* Should be thrown to pass the test */
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -394,15 +418,16 @@ public void testReaderKeyIteration() {
|
||||
final String TEST_METHOD_KEY = "testReaderKeyIteration.mapfile";
|
||||
int SIZE = 10;
|
||||
int ITERATIONS = 5;
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
|
||||
Text.class);
|
||||
writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
|
||||
int start = 0;
|
||||
for (int i = 0; i < SIZE; i++)
|
||||
writer.append(new IntWritable(i), new Text("Value:" + i));
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
|
||||
reader = createReader(TEST_METHOD_KEY, IntWritable.class);
|
||||
// test iteration
|
||||
Writable startValue = new Text("Value:" + start);
|
||||
int i = 0;
|
||||
@ -421,6 +446,8 @@ public void testReaderKeyIteration() {
|
||||
reader.seek(new IntWritable(SIZE * 2)));
|
||||
} catch (IOException ex) {
|
||||
fail("reader seek error !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -431,11 +458,11 @@ public void testReaderKeyIteration() {
|
||||
public void testFix() {
|
||||
final String INDEX_LESS_MAP_FILE = "testFix.mapfile";
|
||||
int PAIR_SIZE = 20;
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
Path dir = new Path(TEST_DIR, INDEX_LESS_MAP_FILE);
|
||||
MapFile.Writer writer = createWriter(INDEX_LESS_MAP_FILE,
|
||||
IntWritable.class, Text.class);
|
||||
writer = createWriter(INDEX_LESS_MAP_FILE, IntWritable.class, Text.class);
|
||||
for (int i = 0; i < PAIR_SIZE; i++)
|
||||
writer.append(new IntWritable(0), new Text("value"));
|
||||
writer.close();
|
||||
@ -450,6 +477,8 @@ public void testFix() {
|
||||
MapFile.fix(fs, dir, IntWritable.class, Text.class, true, conf) == PAIR_SIZE);
|
||||
} catch (Exception ex) {
|
||||
fail("testFix error !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
/**
|
||||
@ -459,38 +488,46 @@ public void testFix() {
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testDeprecatedConstructors() {
|
||||
String path = new Path(TEST_DIR, "writes.mapfile").toString();
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
MapFile.Writer writer = new MapFile.Writer(conf, fs, path,
|
||||
writer = new MapFile.Writer(conf, fs, path,
|
||||
IntWritable.class, Text.class, CompressionType.RECORD);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
|
||||
Text.class, CompressionType.RECORD, defaultProgressable);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
|
||||
Text.class, CompressionType.RECORD, defaultCodec, defaultProgressable);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
writer = new MapFile.Writer(conf, fs, path,
|
||||
WritableComparator.get(Text.class), Text.class);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
writer = new MapFile.Writer(conf, fs, path,
|
||||
WritableComparator.get(Text.class), Text.class,
|
||||
SequenceFile.CompressionType.RECORD);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
writer = new MapFile.Writer(conf, fs, path,
|
||||
WritableComparator.get(Text.class), Text.class,
|
||||
CompressionType.RECORD, defaultProgressable);
|
||||
assertNotNull(writer);
|
||||
writer.close();
|
||||
|
||||
MapFile.Reader reader = new MapFile.Reader(fs, path,
|
||||
reader = new MapFile.Reader(fs, path,
|
||||
WritableComparator.get(IntWritable.class), conf);
|
||||
assertNotNull(reader);
|
||||
assertNotNull("reader key is null !!!", reader.getKeyClass());
|
||||
assertNotNull("reader value in null", reader.getValueClass());
|
||||
|
||||
} catch (IOException e) {
|
||||
fail(e.getMessage());
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -509,11 +546,7 @@ public void testKeyLessWriterCreation() {
|
||||
} catch (Exception e) {
|
||||
fail("fail in testKeyLessWriterCreation. Other ex !!!");
|
||||
} finally {
|
||||
if (writer != null)
|
||||
try {
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
/**
|
||||
@ -542,11 +575,7 @@ public void testPathExplosionWriterCreation() {
|
||||
} catch (Exception e) {
|
||||
fail("fail in testPathExplosionWriterCreation. Other ex !!!");
|
||||
} finally {
|
||||
if (writer != null)
|
||||
try {
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -555,9 +584,9 @@ public void testPathExplosionWriterCreation() {
|
||||
*/
|
||||
@Test
|
||||
public void testDescOrderWithThrowExceptionWriterAppend() {
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(".mapfile", IntWritable.class,
|
||||
Text.class);
|
||||
writer = createWriter(".mapfile", IntWritable.class, Text.class);
|
||||
writer.append(new IntWritable(2), new Text("value: " + 1));
|
||||
writer.append(new IntWritable(2), new Text("value: " + 2));
|
||||
writer.append(new IntWritable(2), new Text("value: " + 4));
|
||||
@ -566,6 +595,8 @@ public void testDescOrderWithThrowExceptionWriterAppend() {
|
||||
} catch (IOException ex) {
|
||||
} catch (Exception e) {
|
||||
fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -575,15 +606,17 @@ public void testMainMethodMapFile() {
|
||||
String inFile = "mainMethodMapFile.mapfile";
|
||||
String outFile = "mainMethodMapFile.mapfile";
|
||||
String[] args = { path, outFile };
|
||||
MapFile.Writer writer = null;
|
||||
try {
|
||||
MapFile.Writer writer = createWriter(inFile, IntWritable.class,
|
||||
Text.class);
|
||||
writer = createWriter(inFile, IntWritable.class, Text.class);
|
||||
writer.append(new IntWritable(1), new Text("test_text1"));
|
||||
writer.append(new IntWritable(2), new Text("test_text2"));
|
||||
writer.close();
|
||||
MapFile.main(args);
|
||||
} catch (Exception ex) {
|
||||
fail("testMainMethodMapFile error !!!");
|
||||
} finally {
|
||||
IOUtils.cleanup(null, writer);
|
||||
}
|
||||
}
|
||||
|
||||
@ -601,8 +634,11 @@ public void testGetClosest() throws Exception {
|
||||
Path qualifiedDirName = fs.makeQualified(dirName);
|
||||
// Make an index entry for every third insertion.
|
||||
MapFile.Writer.setIndexInterval(conf, 3);
|
||||
MapFile.Writer writer = new MapFile.Writer(conf, fs,
|
||||
qualifiedDirName.toString(), Text.class, Text.class);
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
|
||||
Text.class, Text.class);
|
||||
// Assert that the index interval is 1
|
||||
assertEquals(3, writer.getIndexInterval());
|
||||
// Add entries up to 100 in intervals of ten.
|
||||
@ -614,8 +650,7 @@ public void testGetClosest() throws Exception {
|
||||
}
|
||||
writer.close();
|
||||
// Now do getClosest on created mapfile.
|
||||
MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
|
||||
try {
|
||||
reader = new MapFile.Reader(qualifiedDirName, conf);
|
||||
Text key = new Text("55");
|
||||
Text value = new Text();
|
||||
Text closest = (Text) reader.getClosest(key, value);
|
||||
@ -650,7 +685,7 @@ public void testGetClosest() throws Exception {
|
||||
closest = (Text) reader.getClosest(key, value, true);
|
||||
assertEquals(new Text("90"), closest);
|
||||
} finally {
|
||||
reader.close();
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
@ -662,16 +697,18 @@ public void testMidKey() throws Exception {
|
||||
FileSystem fs = FileSystem.getLocal(conf);
|
||||
Path qualifiedDirName = fs.makeQualified(dirName);
|
||||
|
||||
MapFile.Writer writer = new MapFile.Writer(conf, fs,
|
||||
qualifiedDirName.toString(), IntWritable.class, IntWritable.class);
|
||||
MapFile.Writer writer = null;
|
||||
MapFile.Reader reader = null;
|
||||
try {
|
||||
writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
|
||||
IntWritable.class, IntWritable.class);
|
||||
writer.append(new IntWritable(1), new IntWritable(1));
|
||||
writer.close();
|
||||
// Now do getClosest on created mapfile.
|
||||
MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
|
||||
try {
|
||||
reader = new MapFile.Reader(qualifiedDirName, conf);
|
||||
assertEquals(new IntWritable(1), reader.midKey());
|
||||
} finally {
|
||||
reader.close();
|
||||
IOUtils.cleanup(null, writer, reader);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user