From 905bfa84a8b573c0582e0102e5ac23ed6ba1f804 Mon Sep 17 00:00:00 2001 From: WangYuanben <48795318+YuanbenWang@users.noreply.github.com> Date: Fri, 12 May 2023 06:42:02 +0800 Subject: [PATCH] HDFS-16965. Add switch to decide whether to enable native codec. (#5520). Contributed by WangYuanben. Reviewed-by: Tao Li Reviewed-by: Shilun Fan Signed-off-by: Ayush Saxena --- .../hadoop/io/erasurecode/CodecUtil.java | 17 ++++++++ .../src/main/resources/core-default.xml | 10 +++++ .../erasurecode/TestCodecRawCoderMapping.java | 41 ++++++++++++++++++- 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index f89a0d9812..407db29143 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -78,6 +78,11 @@ public final class CodecUtil { public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "xor.rawcoders"; + public static final String IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY = + IO_ERASURECODE_CODEC + "native.enabled"; + + public static final boolean IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT = true; + private CodecUtil() { } /** @@ -170,8 +175,14 @@ private static String[] getRawCoderNames( private static RawErasureEncoder createRawEncoderWithFallback( Configuration conf, String codecName, ErasureCoderOptions coderOptions) { + boolean nativeEncoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY, + IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT); String[] rawCoderNames = getRawCoderNames(conf, codecName); for (String rawCoderName : rawCoderNames) { + if (!nativeEncoderEnabled && rawCoderName.contains("native")) { + LOG.debug("Disable the encoder with ISA-L."); + continue; + } try { if (rawCoderName != null) { RawErasureCoderFactory fact = createRawCoderFactory( @@ -192,8 +203,14 @@ private static RawErasureEncoder createRawEncoderWithFallback( private static RawErasureDecoder createRawDecoderWithFallback( Configuration conf, String codecName, ErasureCoderOptions coderOptions) { + boolean nativeDecoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY, + IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT); String[] coders = getRawCoderNames(conf, codecName); for (String rawCoderName : coders) { + if (!nativeDecoderEnabled && rawCoderName.contains("native")) { + LOG.debug("Disable the decoder with ISA-L."); + continue; + } try { if (rawCoderName != null) { RawErasureCoderFactory fact = createRawCoderFactory( diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index e18a50c72e..537feebaec 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -920,6 +920,16 @@ + + io.erasurecode.codec.native.enabled + true + + Used to decide whether to enable native codec. If set to false, native codec + would not be created and ISA-L support would be disabled. Recommend to set to + false when your CPU does not support ISA-L. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index c7874dc77a..3e7541b8da 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -30,17 +30,21 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + /** * Test the codec to raw coder mapping. */ public class TestCodecRawCoderMapping { - private static Configuration conf; private static final int numDataUnit = 6; private static final int numParityUnit = 3; @@ -150,4 +154,39 @@ public void testIgnoreInvalidCodec() { conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); Assert.assertTrue(decoder instanceof XORRawDecoder); } + + @Test + public void testCodecNativeEnabled() { + assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + ErasureCoderOptions coderOptions = new ErasureCoderOptions( + numDataUnit, numParityUnit); + + RawErasureEncoder rsEncoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + RawErasureDecoder rsDecoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + RawErasureEncoder xorEncoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + RawErasureDecoder xorDecoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + assertTrue(rsEncoder instanceof NativeRSRawEncoder); + assertTrue(rsDecoder instanceof NativeRSRawDecoder); + assertTrue(xorEncoder instanceof NativeXORRawEncoder); + assertTrue(xorDecoder instanceof NativeXORRawDecoder); + + conf.setBoolean(CodecUtil.IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY, + false); + rsEncoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + rsDecoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); + xorEncoder = CodecUtil.createRawEncoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + xorDecoder = CodecUtil.createRawDecoder( + conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); + assertTrue(rsEncoder instanceof RSRawEncoder); + assertTrue(rsDecoder instanceof RSRawDecoder); + assertTrue(xorEncoder instanceof XORRawEncoder); + assertTrue(xorDecoder instanceof XORRawDecoder); + } }