big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15612. Improve exception when tfile fails to load LzoCodec. (gera)

Browse Source
This commit is contained in:
Gera Shegalov 2018-07-17 00:05:39 -07:00
parent ea2c6c8c9a
commit 6bec03cfc8
2 changed files with 53 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Compression.java
View File

@ -24,6 +24,7 @@
import java.io.OutputStream;
import java.util.ArrayList;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
@ -78,25 +79,33 @@ public void flush() throws IOException {
public enum Algorithm {
LZO(TFile.COMPRESSION_LZO) {
private transient boolean checked = false;
private transient ClassNotFoundException cnf;
private transient boolean reinitCodecInTests;
private static final String defaultClazz =
"org.apache.hadoop.io.compress.LzoCodec";
private transient String clazz;
private transient CompressionCodec codec = null;
@Override
public synchronized boolean isSupported() {
if (!checked) {
checked = true;
private String getLzoCodecClass() {
String extClazzConf = conf.get(CONF_LZO_CLASS);
String extClazz = (extClazzConf != null) ?
extClazzConf : System.getProperty(CONF_LZO_CLASS);
String clazz = (extClazz != null) ? extClazz : defaultClazz;
return (extClazz != null) ? extClazz : defaultClazz;
}
@Override
public synchronized boolean isSupported() {
if (!checked || reinitCodecInTests) {
checked = true;
reinitCodecInTests = conf.getBoolean("test.reload.lzo.codec", false);
clazz = getLzoCodecClass();
try {
LOG.info("Trying to load Lzo codec class: " + clazz);
codec =
(CompressionCodec) ReflectionUtils.newInstance(Class
.forName(clazz), conf);
} catch (ClassNotFoundException e) {
// that is okay
cnf = e;
}
}
return codec != null;
@ -105,9 +114,9 @@ public synchronized boolean isSupported() {
@Override
CompressionCodec getCodec() throws IOException {
if (!isSupported()) {
throw new IOException(
"LZO codec class not specified. Did you forget to set property "
+ CONF_LZO_CLASS + "?");
throw new IOException(String.format(
"LZO codec %s=%s could not be loaded", CONF_LZO_CLASS, clazz),
cnf);
}
return codec;

34
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
View File

@ -17,14 +17,28 @@
*/
package org.apache.hadoop.io.file.tfile;
import org.junit.Test;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.*;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestCompression {
@BeforeClass
public static void resetConfigBeforeAll() {
Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", true);
}
@AfterClass
public static void resetConfigAfterAll() {
Compression.Algorithm.LZO.conf.setBoolean("test.reload.lzo.codec", false);
}
/**
* Regression test for HADOOP-11418.
* Verify we can set a LZO codec different from default LZO codec.
@ -38,4 +52,22 @@ public void testConfigureLZOCodec() throws IOException {
assertEquals(defaultCodec,
Compression.Algorithm.LZO.getCodec().getClass().getName());
}
@Test
public void testMisconfiguredLZOCodec() throws Exception {
// Dummy codec
String defaultCodec = "org.apache.hadoop.io.compress.InvalidLzoCodec";
Compression.Algorithm.conf.set(
Compression.Algorithm.CONF_LZO_CLASS, defaultCodec);
IOException ioEx = LambdaTestUtils.intercept(
IOException.class,
defaultCodec,
() -> Compression.Algorithm.LZO.getCodec());
if (!(ioEx.getCause() instanceof ClassNotFoundException)) {
throw ioEx;
}
}
}