big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16965. Add switch to decide whether to enable native codec. (#5520). Contributed by WangYuanben.

Browse Source 
Reviewed-by: Tao Li <tomscut@apache.org>
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
WangYuanben 2023-05-12 06:42:02 +08:00 committed by GitHub
parent e76c09ac3b
commit 905bfa84a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 67 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
View File

@ -78,6 +78,11 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "xor.rawcoders";
public static final String IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY =
IO_ERASURECODE_CODEC + "native.enabled";
public static final boolean IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT = true;
private CodecUtil() { }
/**
@ -170,8 +175,14 @@ private static String[] getRawCoderNames(
private static RawErasureEncoder createRawEncoderWithFallback(
Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
boolean nativeEncoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT);
String[] rawCoderNames = getRawCoderNames(conf, codecName);
for (String rawCoderName : rawCoderNames) {
if (!nativeEncoderEnabled && rawCoderName.contains("native")) {
LOG.debug("Disable the encoder with ISA-L.");
continue;
}
try {
if (rawCoderName != null) {
RawErasureCoderFactory fact = createRawCoderFactory(
@ -192,8 +203,14 @@ private static RawErasureEncoder createRawEncoderWithFallback(
private static RawErasureDecoder createRawDecoderWithFallback(
Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
boolean nativeDecoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT);
String[] coders = getRawCoderNames(conf, codecName);
for (String rawCoderName : coders) {
if (!nativeDecoderEnabled && rawCoderName.contains("native")) {
LOG.debug("Disable the decoder with ISA-L.");
continue;
}
try {
if (rawCoderName != null) {
RawErasureCoderFactory fact = createRawCoderFactory(

10
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -920,6 +920,16 @@
</description>
</property>
<property>
<name>io.erasurecode.codec.native.enabled</name>
<value>true</value>
<description>
Used to decide whether to enable native codec. If set to false, native codec
would not be created and ISA-L support would be disabled. Recommend to set to
false when your CPU does not support ISA-L.
</description>
</property>
<!-- file system properties -->
<property>

41
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
View File

@ -30,17 +30,21 @@
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
/**
* Test the codec to raw coder mapping.
*/
public class TestCodecRawCoderMapping {
private static Configuration conf;
private static final int numDataUnit = 6;
private static final int numParityUnit = 3;
@ -150,4 +154,39 @@ public void testIgnoreInvalidCodec() {
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof XORRawDecoder);
}
@Test
public void testCodecNativeEnabled() {
assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit);
RawErasureEncoder rsEncoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
RawErasureDecoder rsDecoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
RawErasureEncoder xorEncoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
RawErasureDecoder xorDecoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
assertTrue(rsEncoder instanceof NativeRSRawEncoder);
assertTrue(rsDecoder instanceof NativeRSRawDecoder);
assertTrue(xorEncoder instanceof NativeXORRawEncoder);
assertTrue(xorDecoder instanceof NativeXORRawDecoder);
conf.setBoolean(CodecUtil.IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
false);
rsEncoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
rsDecoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
xorEncoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
xorDecoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
assertTrue(rsEncoder instanceof RSRawEncoder);
assertTrue(rsDecoder instanceof RSRawDecoder);
assertTrue(xorEncoder instanceof XORRawEncoder);
assertTrue(xorDecoder instanceof XORRawDecoder);
}
}