big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11658. Externalize io.compression.codecs property. Contributed by Kai Zheng.

Browse Source
This commit is contained in:
Akira Ajisaka 2015-03-02 01:09:54 -08:00
parent cbb492578e
commit ca1c00bf81
4 changed files with 33 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -642,6 +642,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-10976. moving the source code of hadoop-tools docs to the
directory under hadoop-tools (Masatake Iwasaki via aw)
HADOOP-11658. Externalize io.compression.codecs property.
(Kai Zheng via aajisaka)
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -91,17 +91,24 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final String IPC_CALLQUEUE_IMPL_KEY = "callqueue.impl";
public static final String IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY = "identity-provider.impl";
/** Internal buffer size for Lzo compressor/decompressors */
public static final String IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY =
"io.compression.codec.lzo.buffersize";
/** Default value for IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY */
public static final int IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_DEFAULT =
64*1024;
/** This is for specifying the implementation for the mappings from
* hostnames to the racks they belong to
*/
public static final String NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY =
"net.topology.configured.node.mapping";
"net.topology.configured.node.mapping";
/**
* Supported compression codec classes
*/
public static final String IO_COMPRESSION_CODECS_KEY = "io.compression.codecs";
/** Internal buffer size for Lzo compressor/decompressors */
public static final String IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY =
"io.compression.codec.lzo.buffersize";
/** Default value for IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY */
public static final int IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_DEFAULT =
64*1024;
/** Internal buffer size for Snappy compressor/decompressors */
public static final String IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY =

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
@ -106,7 +107,8 @@ public String toString() {
* @param conf the configuration to look in
* @return a list of the {@link CompressionCodec} classes
*/
public static List<Class<? extends CompressionCodec>> getCodecClasses(Configuration conf) {
public static List<Class<? extends CompressionCodec>> getCodecClasses(
Configuration conf) {
List<Class<? extends CompressionCodec>> result
= new ArrayList<Class<? extends CompressionCodec>>();
// Add codec classes discovered via service loading
@ -118,7 +120,8 @@ public static List<Class<? extends CompressionCodec>> getCodecClasses(Configurat
}
}
// Add codec classes from configuration
String codecsString = conf.get("io.compression.codecs");
String codecsString = conf.get(
CommonConfigurationKeys.IO_COMPRESSION_CODECS_KEY);
if (codecsString != null) {
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
@ -161,7 +164,7 @@ public static void setCodecClasses(Configuration conf,
buf.append(itr.next().getName());
}
}
conf.set("io.compression.codecs", buf.toString());
conf.set(CommonConfigurationKeys.IO_COMPRESSION_CODECS_KEY, buf.toString());
}
/**
@ -172,7 +175,8 @@ public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
codecsByClassName = new HashMap<String, CompressionCodec>();
codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf);
List<Class<? extends CompressionCodec>> codecClasses =
getCodecClasses(conf);
if (codecClasses == null || codecClasses.isEmpty()) {
addCodec(new GzipCodec());
addCodec(new DefaultCodec());
@ -193,7 +197,8 @@ public CompressionCodec getCodec(Path file) {
CompressionCodec result = null;
if (codecs != null) {
String filename = file.getName();
String reversedFilename = new StringBuilder(filename).reverse().toString();
String reversedFilename =
new StringBuilder(filename).reverse().toString();
SortedMap<String, CompressionCodec> subMap =
codecs.headMap(reversedFilename);
if (!subMap.isEmpty()) {
@ -239,7 +244,8 @@ public CompressionCodec getCodecByName(String codecName) {
}
CompressionCodec codec = getCodecByClassName(codecName);
if (codec == null) {
// trying to get the codec by name in case the name was specified instead a class
// trying to get the codec by name in case the name was specified
// instead a class
codec = codecsByName.get(codecName.toLowerCase());
}
return codec;
@ -260,7 +266,8 @@ public CompressionCodec getCodecByName(String codecName) {
* @param codecName the canonical class name of the codec
* @return the codec class
*/
public Class<? extends CompressionCodec> getCodecClassByName(String codecName) {
public Class<? extends CompressionCodec> getCodecClassByName(
String codecName) {
CompressionCodec codec = getCodecByName(codecName);
if (codec == null) {
return null;

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
View File

@ -24,6 +24,7 @@
import java.util.*;
import junit.framework.TestCase;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
@ -258,7 +259,7 @@ public static void testFinding() {
checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
Configuration conf = new Configuration();
conf.set("io.compression.codecs",
conf.set(CommonConfigurationKeys.IO_COMPRESSION_CODECS_KEY,
" org.apache.hadoop.io.compress.GzipCodec , " +
" org.apache.hadoop.io.compress.DefaultCodec , " +
" org.apache.hadoop.io.compress.BZip2Codec ");