diff --git a/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt b/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt index f133f81313..aa65991d2e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt @@ -37,6 +37,9 @@ fs-encryption (Unreleased) HADOOP-10803. Update OpensslCipher#getInstance to accept CipherSuite#name format. (Yi Liu) + HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to + JCE if non native support. (Yi Liu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java index 9c4b8fdd8d..c75311aef6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java @@ -72,4 +72,14 @@ public enum CipherSuite { builder.append("}"); return builder.toString(); } + + public static void checkName(String name) { + CipherSuite[] suites = CipherSuite.values(); + for (CipherSuite suite : suites) { + if (suite.getName().equals(name)) { + return; + } + } + throw new IllegalArgumentException("Invalid cipher suite name: " + name); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java index 80e15cd6b7..45c06f02da 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java @@ -18,13 +18,23 @@ package org.apache.hadoop.crypto; import java.security.GeneralSecurityException; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT; /** * Crypto codec class, encapsulates encryptor/decryptor pair. @@ -32,12 +42,57 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY @InterfaceAudience.Private @InterfaceStability.Evolving public abstract class CryptoCodec implements Configurable { + public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class); public static CryptoCodec getInstance(Configuration conf) { - final Class klass = conf.getClass( - HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class, - CryptoCodec.class); - return ReflectionUtils.newInstance(klass, conf); + List> klasses = getCodecClasses(conf); + String name = conf.get(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT); + CipherSuite.checkName(name); + CryptoCodec codec = null; + for (Class klass : klasses) { + try { + CryptoCodec c = ReflectionUtils.newInstance(klass, conf); + if (c.getCipherSuite().getName().equalsIgnoreCase(name)) { + if (codec == null) { + LOG.debug("Using crypto codec {}.", klass.getName()); + codec = c; + } + } else { + LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.", + klass.getName(), name); + } + } catch (Exception e) { + LOG.warn("Crypto codec {} is not available.", klass.getName()); + } + } + + if (codec != null) { + return codec; + } + + throw new RuntimeException("No available crypto codec which meets " + + "the cipher suite " + name + "."); + } + + private static List> getCodecClasses( + Configuration conf) { + List> result = Lists.newArrayList(); + String codecString = conf.get(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, + HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT); + for (String c : Splitter.on(',').trimResults().omitEmptyStrings(). + split(codecString)) { + try { + Class cls = conf.getClassByName(c); + result.add(cls.asSubclass(CryptoCodec.class)); + } catch (ClassCastException e) { + LOG.warn("Class " + c + " is not a CryptoCodec."); + } catch (ClassNotFoundException e) { + LOG.warn("Crypto codec " + c + " not found."); + } + } + + return result; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java index 04c2db09e1..7db7b063e5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java @@ -47,6 +47,9 @@ public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec { private Random random; public OpensslAesCtrCryptoCodec() { + if (!OpensslCipher.isNativeCodeLoaded()) { + throw new RuntimeException("Failed to load OpenSSL Cipher."); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 9691539821..35c3bce6c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -285,6 +285,14 @@ public class CommonConfigurationKeysPublic { /** See core-default.xml */ public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY = "hadoop.security.crypto.codec.class"; + public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT = + "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec," + + "org.apache.hadoop.crypto.JceAesCtrCryptoCodec"; + /** See core-default.xml */ + public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY = + "hadoop.security.crypto.cipher.suite"; + public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT = + "AES/CTR/NoPadding"; /** See core-default.xml */ public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY = "hadoop.security.crypto.jce.provider"; @@ -302,8 +310,10 @@ public class CommonConfigurationKeysPublic { /** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */ public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT = "SHA1PRNG"; + /** See core-default.xml */ public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY = "hadoop.security.secure.random.impl"; + /** See core-default.xml */ public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY = "hadoop.security.random.device.file.path"; public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 224fb12e92..3c93ab8f11 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1453,10 +1453,20 @@ for ldap providers in the same way as above does. hadoop.security.crypto.codec.class - + org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, + org.apache.hadoop.crypto.JceAesCtrCryptoCodec - The default implementation of CryptoCodec which is used for encryption - and decryption. + Comma list of CryptoCodec implementations which are used for encryption + and decryption. The first implementation will be used if avaiable, others + are fallbacks. + + + + + hadoop.security.crypto.cipher.suite + AES/CTR/NoPadding + + Cipher suite for crypto codec. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java index 8150d57ee6..4e962cb42d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java @@ -29,7 +29,8 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec public static void init() throws Exception { Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, - OpensslAesCtrCryptoCodec.class.getName()); + OpensslAesCtrCryptoCodec.class.getName() + "," + + JceAesCtrCryptoCodec.class.getName()); codec = CryptoCodec.getInstance(conf); } }