HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to JCE if non native support. (yliu)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1610887 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt

@@ -37,6 +37,9 @@ fs-encryption (Unreleased)
     HADOOP-10803. Update OpensslCipher#getInstance to accept CipherSuite#name
     format. (Yi Liu)
 
+    HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to
+    JCE if non native support. (Yi Liu)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java

@@ -72,4 +72,14 @@ public enum CipherSuite {
     builder.append("}");
     return builder.toString();
   }
+  
+  public static void checkName(String name) {
+    CipherSuite[] suites = CipherSuite.values();
+    for (CipherSuite suite : suites) {
+      if (suite.getName().equals(name)) {
+        return;
+      }
+    }
+    throw new IllegalArgumentException("Invalid cipher suite name: " + name);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java

@@ -18,13 +18,23 @@
 package org.apache.hadoop.crypto;
 
 import java.security.GeneralSecurityException;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT;
 
 /**
  * Crypto codec class, encapsulates encryptor/decryptor pair.
@@ -32,12 +42,57 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public abstract class CryptoCodec implements Configurable {
+  public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class);
   
   public static CryptoCodec getInstance(Configuration conf) {
-    final Class<? extends CryptoCodec> klass = conf.getClass(
+    List<Class<? extends CryptoCodec>> klasses = getCodecClasses(conf);
-        HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class, 
+    String name = conf.get(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, 
-        CryptoCodec.class);
+        HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT);
-    return ReflectionUtils.newInstance(klass, conf);
+    CipherSuite.checkName(name);
+    CryptoCodec codec = null;
+    for (Class<? extends CryptoCodec> klass : klasses) {
+      try {
+        CryptoCodec c = ReflectionUtils.newInstance(klass, conf);
+        if (c.getCipherSuite().getName().equalsIgnoreCase(name)) {
+          if (codec == null) {
+            LOG.debug("Using crypto codec {}.", klass.getName());
+            codec = c;
+          }
+        } else {
+          LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.", 
+              klass.getName(), name);
+        }
+      } catch (Exception e) {
+        LOG.warn("Crypto codec {} is not available.", klass.getName());
+      }
+    }
+    
+    if (codec != null) {
+      return codec;
+    }
+    
+    throw new RuntimeException("No available crypto codec which meets " + 
+        "the cipher suite " + name + ".");
+  }
+  
+  private static List<Class<? extends CryptoCodec>> getCodecClasses(
+      Configuration conf) {
+    List<Class<? extends CryptoCodec>> result = Lists.newArrayList();
+    String codecString = conf.get(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY,
+        HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT);
+    for (String c : Splitter.on(',').trimResults().omitEmptyStrings().
+        split(codecString)) {
+      try {
+        Class<?> cls = conf.getClassByName(c);
+        result.add(cls.asSubclass(CryptoCodec.class));
+      } catch (ClassCastException e) {
+        LOG.warn("Class " + c + " is not a CryptoCodec.");
+      } catch (ClassNotFoundException e) {
+        LOG.warn("Crypto codec " + c + " not found.");
+      }
+    }
+    
+    return result;
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java

@@ -47,6 +47,9 @@ public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec {
   private Random random;
   
   public OpensslAesCtrCryptoCodec() {
+    if (!OpensslCipher.isNativeCodeLoaded()) {
+      throw new RuntimeException("Failed to load OpenSSL Cipher.");
+    }
   }
 
   @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -285,6 +285,14 @@ public class CommonConfigurationKeysPublic {
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY =
     "hadoop.security.crypto.codec.class";
+  public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT = 
+    "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec," +
+    "org.apache.hadoop.crypto.JceAesCtrCryptoCodec";
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY =
+    "hadoop.security.crypto.cipher.suite";
+  public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT = 
+    "AES/CTR/NoPadding";
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
       "hadoop.security.crypto.jce.provider";
@@ -302,8 +310,10 @@ public class CommonConfigurationKeysPublic {
   /** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
   public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT = 
       "SHA1PRNG";
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY = 
       "hadoop.security.secure.random.impl";
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY = 
       "hadoop.security.random.device.file.path";
   public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = 

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1453,10 +1453,20 @@ for ldap providers in the same way as above does.
 
 <property>
   <name>hadoop.security.crypto.codec.class</name>
-  <value></value>
+  <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,
+  org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
   <description>
-    The default implementation of CryptoCodec which is used for encryption 
+    Comma list of CryptoCodec implementations which are used for encryption 
-    and decryption. 
+    and decryption. The first implementation will be used if avaiable, others
+    are fallbacks.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.crypto.cipher.suite</name>
+  <value>AES/CTR/NoPadding</value>
+  <description>
+    Cipher suite for crypto codec.
   </description>
 </property>
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java

@@ -29,7 +29,8 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
   public static void init() throws Exception {
     Configuration conf = new Configuration();
     conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, 
-        OpensslAesCtrCryptoCodec.class.getName());
+        OpensslAesCtrCryptoCodec.class.getName() + "," + 
+            JceAesCtrCryptoCodec.class.getName());
     codec = CryptoCodec.getInstance(conf);
   }
 }