From 2a509117344a6b348aa418d8d426cbc12aefb999 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Thu, 8 Aug 2024 21:03:05 +0900 Subject: [PATCH] HADOOP-17609. Make SM4 support optional for OpenSSL native code. (#3019) Reviewed-by: Steve Loughran Reviewed-by: Wei-Chiu Chuang --- .../apache/hadoop/crypto/OpensslCipher.java | 16 ++++++++++++ .../crypto/OpensslSm4CtrCryptoCodec.java | 4 +++ .../org/apache/hadoop/crypto/OpensslCipher.c | 26 ++++++++++++++++++- .../apache/hadoop/crypto/TestCryptoCodec.java | 13 +++------- ...toStreamsWithOpensslSm4CtrCryptoCodec.java | 2 ++ .../hadoop/crypto/TestOpensslCipher.java | 10 +++++++ 6 files changed, 60 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java index b166cfc861..c8a10404b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java @@ -177,6 +177,20 @@ private static Transform tokenizeTransformation(String transformation) } return new Transform(parts[0], parts[1], parts[2]); } + + public static boolean isSupported(CipherSuite suite) { + Transform transform; + int algMode; + int padding; + try { + transform = tokenizeTransformation(suite.getName()); + algMode = AlgMode.get(transform.alg, transform.mode); + padding = Padding.get(transform.padding); + } catch (NoSuchAlgorithmException|NoSuchPaddingException e) { + return false; + } + return isSupportedSuite(algMode, padding); + } /** * Initialize this cipher with a key and IV. @@ -298,5 +312,7 @@ private native int doFinal(long context, ByteBuffer output, int offset, private native void clean(long ctx, long engineNum); + private native static boolean isSupportedSuite(int alg, int padding); + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java index f6b2f6a802..9df1bbe89e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslSm4CtrCryptoCodec.java @@ -41,6 +41,10 @@ public OpensslSm4CtrCryptoCodec() { if (loadingFailureReason != null) { throw new RuntimeException(loadingFailureReason); } + + if (!OpensslCipher.isSupported(CipherSuite.SM4_CTR_NOPADDING)) { + throw new RuntimeException("The OpenSSL native library is built without SM4 CTR support"); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c index f60a19a662..976bf135ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c @@ -232,7 +232,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs #endif loadAesCtr(env); +#if !defined(OPENSSL_NO_SM4) loadSm4Ctr(env); +#endif + #if OPENSSL_VERSION_NUMBER >= 0x10101001L int ret = dlsym_OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); if(!ret) { @@ -245,7 +248,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs if (jthr) { (*env)->DeleteLocalRef(env, jthr); THROW(env, "java/lang/UnsatisfiedLinkError", \ - "Cannot find AES-CTR/SM4-CTR support, is your version of Openssl new enough?"); + "Cannot find AES-CTR support, is your version of OpenSSL new enough?"); return; } } @@ -554,3 +557,24 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibrary } #endif } + +JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_isSupportedSuite + (JNIEnv *env, jclass clazz, jint alg, jint padding) +{ + if (padding != NOPADDING) { + return JNI_FALSE; + } + + if (alg == AES_CTR && (dlsym_EVP_aes_256_ctr != NULL && dlsym_EVP_aes_128_ctr != NULL)) { + return JNI_TRUE; + } + + if (alg == SM4_CTR) { +#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_SM4) + if (dlsym_EVP_sm4_ctr != NULL) { + return JNI_TRUE; + } +#endif + } + return JNI_FALSE; +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index c0fdc51b13..c5b493390a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -106,31 +106,21 @@ public void testJceAesCtrCryptoCodec() throws Exception { @Test(timeout=120000) public void testJceSm4CtrCryptoCodec() throws Exception { - GenericTestUtils.assumeInNativeProfile(); - if (!NativeCodeLoader.buildSupportsOpenssl()) { - LOG.warn("Skipping test since openSSL library not loaded"); - Assume.assumeTrue(false); - } conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, JceSm4CtrCryptoCodec.class.getName()); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, BouncyCastleProvider.PROVIDER_NAME); - Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); cryptoCodecTest(conf, seed, 0, jceSm4CodecClass, jceSm4CodecClass, iv); cryptoCodecTest(conf, seed, count, jceSm4CodecClass, jceSm4CodecClass, iv); - cryptoCodecTest(conf, seed, count, - jceSm4CodecClass, opensslSm4CodecClass, iv); // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff for(int i = 0; i < 8; i++) { iv[8 + i] = (byte) 0xff; } cryptoCodecTest(conf, seed, count, jceSm4CodecClass, jceSm4CodecClass, iv); - cryptoCodecTest(conf, seed, count, - jceSm4CodecClass, opensslSm4CodecClass, iv); } @Test(timeout=120000) @@ -164,6 +154,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { LOG.warn("Skipping test since openSSL library not loaded"); Assume.assumeTrue(false); } + Assume.assumeTrue(OpensslCipher.isSupported(CipherSuite.SM4_CTR_NOPADDING)); conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, BouncyCastleProvider.PROVIDER_NAME); Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); @@ -181,6 +172,8 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception { opensslSm4CodecClass, opensslSm4CodecClass, iv); cryptoCodecTest(conf, seed, count, opensslSm4CodecClass, jceSm4CodecClass, iv); + cryptoCodecTest(conf, seed, count, + jceSm4CodecClass, opensslSm4CodecClass, iv); } private void cryptoCodecTest(Configuration conf, int seed, int count, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java index f634555721..ebc91959e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslSm4CtrCryptoCodec.java @@ -21,6 +21,7 @@ import org.apache.hadoop.crypto.random.OsSecureRandom; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; @@ -40,6 +41,7 @@ public class TestCryptoStreamsWithOpensslSm4CtrCryptoCodec @BeforeClass public static void init() throws Exception { GenericTestUtils.assumeInNativeProfile(); + Assume.assumeTrue(OpensslCipher.isSupported(CipherSuite.SM4_CTR_NOPADDING)); Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding"); conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java index 966a88723a..ff12f3cfe3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java @@ -107,4 +107,14 @@ public void testDoFinalArguments() throws Exception { "Direct buffer is required", e); } } + + @Test(timeout=120000) + public void testIsSupportedSuite() throws Exception { + Assume.assumeTrue("Skipping due to falilure of loading OpensslCipher.", + OpensslCipher.getLoadingFailureReason() == null); + Assert.assertFalse("Unknown suite must not be supported.", + OpensslCipher.isSupported(CipherSuite.UNKNOWN)); + Assert.assertTrue("AES/CTR/NoPadding is not an optional suite.", + OpensslCipher.isSupported(CipherSuite.AES_CTR_NOPADDING)); + } }