HDFS-16965. Add switch to decide whether to enable native codec. (#5520). Contributed by WangYuanben.

Reviewed-by: Tao Li <tomscut@apache.org>
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java

@@ -78,6 +78,11 @@ public final class CodecUtil {
   public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
       IO_ERASURECODE_CODEC + "xor.rawcoders";
 
+  public static final String IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY =
+      IO_ERASURECODE_CODEC + "native.enabled";
+
+  public static final boolean IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT = true;
+
   private CodecUtil() { }
 
   /**
@@ -170,8 +175,14 @@ private static String[] getRawCoderNames(
 
   private static RawErasureEncoder createRawEncoderWithFallback(
       Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
+    boolean nativeEncoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
+        IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT);
     String[] rawCoderNames = getRawCoderNames(conf, codecName);
     for (String rawCoderName : rawCoderNames) {
+      if (!nativeEncoderEnabled && rawCoderName.contains("native")) {
+        LOG.debug("Disable the encoder with ISA-L.");
+        continue;
+      }
       try {
         if (rawCoderName != null) {
           RawErasureCoderFactory fact = createRawCoderFactory(
@@ -192,8 +203,14 @@ private static RawErasureEncoder createRawEncoderWithFallback(
 
   private static RawErasureDecoder createRawDecoderWithFallback(
       Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
+    boolean nativeDecoderEnabled = conf.getBoolean(IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
+        IO_ERASURECODE_CODEC_NATIVE_ENABLED_DEFAULT);
     String[] coders = getRawCoderNames(conf, codecName);
     for (String rawCoderName : coders) {
+      if (!nativeDecoderEnabled && rawCoderName.contains("native")) {
+        LOG.debug("Disable the decoder with ISA-L.");
+        continue;
+      }
       try {
         if (rawCoderName != null) {
           RawErasureCoderFactory fact = createRawCoderFactory(

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -920,6 +920,16 @@
   </description>
 </property>
 
+<property>
+  <name>io.erasurecode.codec.native.enabled</name>
+  <value>true</value>
+  <description>
+    Used to decide whether to enable native codec. If set to false, native codec
+    would not be created and ISA-L support would be disabled. Recommend to set to
+    false when your CPU does not support ISA-L.
+  </description>
+</property>
+
   <!-- file system properties -->
 
 <property>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java

@@ -30,17 +30,21 @@
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
 /**
  * Test the codec to raw coder mapping.
  */
 public class TestCodecRawCoderMapping {
-
   private static Configuration conf;
   private static final int numDataUnit = 6;
   private static final int numParityUnit = 3;
@@ -150,4 +154,39 @@ public void testIgnoreInvalidCodec() {
             conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
     Assert.assertTrue(decoder instanceof XORRawDecoder);
   }
+
+  @Test
+  public void testCodecNativeEnabled() {
+    assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
+    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+        numDataUnit, numParityUnit);
+
+    RawErasureEncoder rsEncoder = CodecUtil.createRawEncoder(
+        conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    RawErasureDecoder rsDecoder = CodecUtil.createRawDecoder(
+        conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    RawErasureEncoder xorEncoder = CodecUtil.createRawEncoder(
+        conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    RawErasureDecoder xorDecoder = CodecUtil.createRawDecoder(
+        conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    assertTrue(rsEncoder instanceof NativeRSRawEncoder);
+    assertTrue(rsDecoder instanceof NativeRSRawDecoder);
+    assertTrue(xorEncoder instanceof NativeXORRawEncoder);
+    assertTrue(xorDecoder instanceof NativeXORRawDecoder);
+
+    conf.setBoolean(CodecUtil.IO_ERASURECODE_CODEC_NATIVE_ENABLED_KEY,
+        false);
+    rsEncoder = CodecUtil.createRawEncoder(
+        conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    rsDecoder = CodecUtil.createRawDecoder(
+        conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    xorEncoder = CodecUtil.createRawEncoder(
+        conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    xorDecoder = CodecUtil.createRawDecoder(
+        conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    assertTrue(rsEncoder instanceof RSRawEncoder);
+    assertTrue(rsDecoder instanceof RSRawDecoder);
+    assertTrue(xorEncoder instanceof XORRawEncoder);
+    assertTrue(xorDecoder instanceof XORRawDecoder);
+  }
 }