From bfef9807a08435822546dd66dc72531465f34743 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 3 Jul 2014 23:40:31 +0000 Subject: [PATCH] HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL (hitliuyi via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1607768 13f79535-47bb-0310-9956-ffa450edef68 --- BUILDING.txt | 21 ++ hadoop-common-project/hadoop-common/pom.xml | 17 +- .../hadoop-common/src/CMakeLists.txt | 33 +++ .../hadoop-common/src/config.h.cmake | 1 + .../hadoop/crypto/AESCTRCryptoCodec.java | 71 ------- .../org/apache/hadoop/crypto/CryptoCodec.java | 2 +- .../hadoop/crypto/JCEAESCTRCryptoCodec.java | 159 -------------- .../apache/hadoop/util/NativeCodeLoader.java | 5 + .../hadoop/util/NativeLibraryChecker.java | 19 +- .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../apache/hadoop/crypto/TestCryptoCodec.java | 195 ++++++++++++++---- .../hadoop/util/TestNativeCodeLoader.java | 4 + .../hadoop-hdfs/CHANGES-fs-encryption.txt | 3 + hadoop-project-dist/pom.xml | 12 ++ hadoop-project/pom.xml | 2 + 15 files changed, 272 insertions(+), 282 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index bfb0e0852e..7b99537a26 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -81,6 +81,27 @@ Maven build goals: the final tar file. This option requires that -Dsnappy.lib is also given, and it ignores the -Dsnappy.prefix option. + OpenSSL build options: + + OpenSSL includes a crypto library that can be utilized by the native code. + It is currently an optional component, meaning that Hadoop can be built with + or without this dependency. + + * Use -Drequire.openssl to fail the build if libcrypto.so is not found. + If this option is not specified and the openssl library is missing, + we silently build a version of libhadoop.so that cannot make use of + openssl. This option is recommended if you plan on making use of openssl + and want to get more repeatable builds. + * Use -Dopenssl.prefix to specify a nonstandard location for the libcrypto + header files and library files. You do not need this option if you have + installed openssl using a package manager. + * Use -Dopenssl.lib to specify a nonstandard location for the libcrypto library + files. Similarly to openssl.prefix, you do not need this option if you have + installed openssl using a package manager. + * Use -Dbundle.openssl to copy the contents of the openssl.lib directory into + the final tar file. This option requires that -Dopenssl.lib is also given, + and it ignores the -Dopenssl.prefix option. + Tests options: * Use -DskipTests to skip tests when running the following Maven goals: diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index ce14b21c38..13b0c3b369 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -483,6 +483,10 @@ false + + + + false @@ -532,6 +536,7 @@ org.apache.hadoop.io.compress.snappy.SnappyDecompressor org.apache.hadoop.io.compress.lz4.Lz4Compressor org.apache.hadoop.io.compress.lz4.Lz4Decompressor + org.apache.hadoop.crypto.OpensslCipher org.apache.hadoop.util.NativeCrc32 org.apache.hadoop.net.unix.DomainSocket org.apache.hadoop.net.unix.DomainSocketWatcher @@ -552,7 +557,7 @@ - + @@ -596,6 +601,11 @@ false true + + + + false + true @@ -641,6 +651,7 @@ org.apache.hadoop.io.compress.snappy.SnappyDecompressor org.apache.hadoop.io.compress.lz4.Lz4Compressor org.apache.hadoop.io.compress.lz4.Lz4Decompressor + org.apache.hadoop.crypto.OpensslCipher org.apache.hadoop.util.NativeCrc32 ${project.build.directory}/native/javah @@ -685,6 +696,10 @@ /p:CustomSnappyLib=${snappy.lib} /p:CustomSnappyInclude=${snappy.include} /p:RequireSnappy=${require.snappy} + /p:CustomOpensslPrefix=${openssl.prefix} + /p:CustomOpensslLib=${openssl.lib} + /p:CustomOpensslInclude=${openssl.include} + /p:RequireOpenssl=${require.openssl} diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index dec63c45d7..9ad049e085 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -145,6 +145,37 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) ENDIF(REQUIRE_SNAPPY) endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) +SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) +set_find_shared_library_version("1.0.0") +SET(OPENSSL_NAME "crypto") +IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + SET(OPENSSL_NAME "eay32") +ENDIF() +find_library(OPENSSL_LIBRARY + NAMES ${OPENSSL_NAME} + PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib + ${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH) +find_library(OPENSSL_LIBRARY + NAMES ${OPENSSL_NAME}) +SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) +find_path(OPENSSL_INCLUDE_DIR + NAMES openssl/evp.h + PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include + ${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH) +find_path(OPENSSL_INCLUDE_DIR + NAMES openssl/evp.h) +if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR) + GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME) + SET(OPENSSL_SOURCE_FILES + "${D}/crypto/OpensslCipher.c") +else (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR) + SET(OPENSSL_INCLUDE_DIR "") + SET(OPENSSL_SOURCE_FILES "") + IF(REQUIRE_OPENSSL) + MESSAGE(FATAL_ERROR "Required openssl library could not be found. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_INCLUDE_DIR=${CUSTOM_OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}") + ENDIF(REQUIRE_OPENSSL) +endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR) + include_directories( ${GENERATED_JAVAH} main/native/src @@ -155,6 +186,7 @@ include_directories( ${ZLIB_INCLUDE_DIRS} ${BZIP2_INCLUDE_DIR} ${SNAPPY_INCLUDE_DIR} + ${OPENSSL_INCLUDE_DIR} ${D}/util ) CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) @@ -172,6 +204,7 @@ add_dual_library(hadoop ${D}/io/compress/lz4/lz4.c ${D}/io/compress/lz4/lz4hc.c ${SNAPPY_SOURCE_FILES} + ${OPENSSL_SOURCE_FILES} ${D}/io/compress/zlib/ZlibCompressor.c ${D}/io/compress/zlib/ZlibDecompressor.c ${BZIP2_SOURCE_FILES} diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake index 020017c02f..d71271dd3e 100644 --- a/hadoop-common-project/hadoop-common/src/config.h.cmake +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -21,6 +21,7 @@ #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" #cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@" #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" +#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@" #cmakedefine HAVE_SYNC_FILE_RANGE #cmakedefine HAVE_POSIX_FADVISE diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java index e26135d266..e69de29bb2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.crypto; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import com.google.common.base.Preconditions; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public abstract class AESCTRCryptoCodec extends CryptoCodec { - - protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING; - - /** - * For AES, the algorithm block is fixed size of 128 bits. - * @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard - */ - private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize(); - private static final int CTR_OFFSET = 8; - - @Override - public CipherSuite getCipherSuite() { - return SUITE; - } - - /** - * The IV is produced by adding the initial IV to the counter. IV length - * should be the same as {@link #AES_BLOCK_SIZE} - */ - @Override - public void calculateIV(byte[] initIV, long counter, byte[] IV) { - Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE); - Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE); - - System.arraycopy(initIV, 0, IV, 0, CTR_OFFSET); - long l = (initIV[CTR_OFFSET + 0] << 56) - + ((initIV[CTR_OFFSET + 1] & 0xFF) << 48) - + ((initIV[CTR_OFFSET + 2] & 0xFF) << 40) - + ((initIV[CTR_OFFSET + 3] & 0xFF) << 32) - + ((initIV[CTR_OFFSET + 4] & 0xFF) << 24) - + ((initIV[CTR_OFFSET + 5] & 0xFF) << 16) - + ((initIV[CTR_OFFSET + 6] & 0xFF) << 8) - + (initIV[CTR_OFFSET + 7] & 0xFF); - l += counter; - IV[CTR_OFFSET + 0] = (byte) (l >>> 56); - IV[CTR_OFFSET + 1] = (byte) (l >>> 48); - IV[CTR_OFFSET + 2] = (byte) (l >>> 40); - IV[CTR_OFFSET + 3] = (byte) (l >>> 32); - IV[CTR_OFFSET + 4] = (byte) (l >>> 24); - IV[CTR_OFFSET + 5] = (byte) (l >>> 16); - IV[CTR_OFFSET + 6] = (byte) (l >>> 8); - IV[CTR_OFFSET + 7] = (byte) (l); - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java index 7d4e65ba4d..80e15cd6b7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java @@ -35,7 +35,7 @@ public abstract class CryptoCodec implements Configurable { public static CryptoCodec getInstance(Configuration conf) { final Class klass = conf.getClass( - HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JCEAESCTRCryptoCodec.class, + HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class, CryptoCodec.class); return ReflectionUtils.newInstance(klass, conf); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java index e575e5ed66..e69de29bb2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.crypto; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.GeneralSecurityException; -import java.security.SecureRandom; - -import javax.crypto.Cipher; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; - -import com.google.common.base.Preconditions; - -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT; - -/** - * Implement the AES-CTR crypto codec using JCE provider. - */ -@InterfaceAudience.Private -public class JCEAESCTRCryptoCodec extends AESCTRCryptoCodec { - private Configuration conf; - private String provider; - private SecureRandom random; - - public JCEAESCTRCryptoCodec() { - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY); - final String secureRandomAlg = conf.get( - HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY, - HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT); - try { - random = (provider != null) ? - SecureRandom.getInstance(secureRandomAlg, provider) : - SecureRandom.getInstance(secureRandomAlg); - } catch (GeneralSecurityException e) { - throw new IllegalArgumentException(e); - } - } - - @Override - public Encryptor createEncryptor() throws GeneralSecurityException { - return new JCEAESCTRCipher(Cipher.ENCRYPT_MODE, provider); - } - - @Override - public Decryptor createDecryptor() throws GeneralSecurityException { - return new JCEAESCTRCipher(Cipher.DECRYPT_MODE, provider); - } - - @Override - public void generateSecureRandom(byte[] bytes) { - random.nextBytes(bytes); - } - - private static class JCEAESCTRCipher implements Encryptor, Decryptor { - private final Cipher cipher; - private final int mode; - private boolean contextReset = false; - - public JCEAESCTRCipher(int mode, String provider) - throws GeneralSecurityException { - this.mode = mode; - if (provider == null || provider.isEmpty()) { - cipher = Cipher.getInstance(SUITE.getName()); - } else { - cipher = Cipher.getInstance(SUITE.getName(), provider); - } - } - - @Override - public void init(byte[] key, byte[] iv) throws IOException { - Preconditions.checkNotNull(key); - Preconditions.checkNotNull(iv); - contextReset = false; - try { - cipher.init(mode, new SecretKeySpec(key, "AES"), - new IvParameterSpec(iv)); - } catch (Exception e) { - throw new IOException(e); - } - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to encrypt entire input buffer. - */ - @Override - public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - /** - * AES-CTR will consume all of the input data. It requires enough space in - * the destination buffer to decrypt entire input buffer. - */ - @Override - public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - process(inBuffer, outBuffer); - } - - private void process(ByteBuffer inBuffer, ByteBuffer outBuffer) - throws IOException { - try { - int inputSize = inBuffer.remaining(); - // Cipher#update will maintain crypto context. - int n = cipher.update(inBuffer, outBuffer); - if (n < inputSize) { - /** - * Typically code will not get here. Cipher#update will consume all - * input data and put result in outBuffer. - * Cipher#doFinal will reset the crypto context. - */ - contextReset = true; - cipher.doFinal(inBuffer, outBuffer); - } - } catch (Exception e) { - throw new IOException(e); - } - } - - @Override - public boolean isContextReset() { - return contextReset; - } - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index 5667d98b3e..533fc07f8d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -78,6 +78,11 @@ public static boolean isNativeCodeLoaded() { * Returns true only if this build was compiled with support for snappy. */ public static native boolean buildSupportsSnappy(); + + /** + * Returns true only if this build was compiled with support for openssl. + */ + public static native boolean buildSupportsOpenssl(); public static native String getLibraryName(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index 84117e2002..4891f03cbb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -20,6 +20,7 @@ import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; @@ -57,12 +58,14 @@ public static void main(String[] args) { boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; boolean snappyLoaded = false; + boolean opensslLoaded = false; // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; + String opensslLibraryName = ""; String lz4LibraryName = ""; String bzip2LibraryName = ""; if (nativeHadoopLoaded) { @@ -76,6 +79,11 @@ public static void main(String[] args) { if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { snappyLibraryName = SnappyCodec.getLibraryName(); } + opensslLoaded = NativeCodeLoader.buildSupportsOpenssl() && + OpensslCipher.isNativeCodeLoaded(); + if (opensslLoaded) { + opensslLibraryName = OpensslCipher.getLibraryName(); + } if (lz4Loaded) { lz4LibraryName = Lz4Codec.getLibraryName(); } @@ -84,11 +92,12 @@ public static void main(String[] args) { } } System.out.println("Native library checking:"); - System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); - System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); - System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); - System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); - System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); + System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); + System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); + System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); + System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); + System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); + System.out.printf("openssl: %b %s\n", opensslLoaded, opensslLibraryName); if ((!nativeHadoopLoaded) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c index d03050c591..3625112311 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c @@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup #endif } +JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl + (JNIEnv *env, jclass clazz) +{ +#ifdef HADOOP_OPENSSL_LIBRARY + return JNI_TRUE; +#else + return JNI_FALSE; +#endif +} + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName (JNIEnv *env, jclass clazz) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index f4a34a1854..d95052815c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -17,38 +17,165 @@ */ package org.apache.hadoop.crypto; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.junit.AfterClass; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.RandomDatum; +import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.ReflectionUtils; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; public class TestCryptoCodec { - private static CryptoCodec codec; + private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class); + private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16}; + private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; + private static final int bufferSize = 4096; - @BeforeClass - public static void init() throws Exception { - Configuration conf = new Configuration(); - codec = CryptoCodec.getInstance(conf); - } - - @AfterClass - public static void shutdown() throws Exception { - } + private Configuration conf = new Configuration(); + private int count = 10000; + private int seed = new Random().nextInt(); @Test(timeout=120000) - public void testSecureRandom() throws Exception { - // len = 16 - checkSecureRandom(16); - - // len = 32 - checkSecureRandom(32); - - // len = 128 - checkSecureRandom(128); + public void testJceAesCtrCryptoCodec() throws Exception { + cryptoCodecTest(conf, seed, 0, + "org.apache.hadoop.crypto.JceAesCtrCryptoCodec"); + cryptoCodecTest(conf, seed, count, + "org.apache.hadoop.crypto.JceAesCtrCryptoCodec"); } - private void checkSecureRandom(int len) { + @Test(timeout=1200000) + public void testOpensslAesCtrCryptoCodec() throws Exception { + if (NativeCodeLoader.buildSupportsOpenssl()) { + Assert.assertTrue(OpensslCipher.isNativeCodeLoaded()); + } + if (OpensslCipher.isNativeCodeLoaded()) { + cryptoCodecTest(conf, seed, 0, + "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); + cryptoCodecTest(conf, seed, count, + "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); + } + } + + private void cryptoCodecTest(Configuration conf, int seed, int count, + String codecClass) throws IOException, GeneralSecurityException { + CryptoCodec codec = null; + try { + codec = (CryptoCodec)ReflectionUtils.newInstance( + conf.getClassByName(codecClass), conf); + } catch (ClassNotFoundException cnfe) { + throw new IOException("Illegal crypto codec!"); + } + LOG.info("Created a Codec object of type: " + codecClass); + + // Generate data + DataOutputBuffer data = new DataOutputBuffer(); + RandomDatum.Generator generator = new RandomDatum.Generator(seed); + for(int i = 0; i < count; ++i) { + generator.next(); + RandomDatum key = generator.getKey(); + RandomDatum value = generator.getValue(); + + key.write(data); + value.write(data); + } + LOG.info("Generated " + count + " records"); + + // Encrypt data + DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer(); + CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, + codec, bufferSize, key, iv); + out.write(data.getData(), 0, data.getLength()); + out.flush(); + out.close(); + LOG.info("Finished encrypting data"); + + // Decrypt data + DataInputBuffer decryptedDataBuffer = new DataInputBuffer(); + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + encryptedDataBuffer.getLength()); + CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, + codec, bufferSize, key, iv); + DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in)); + + // Check + DataInputBuffer originalData = new DataInputBuffer(); + originalData.reset(data.getData(), 0, data.getLength()); + DataInputStream originalIn = new DataInputStream( + new BufferedInputStream(originalData)); + + for(int i=0; i < count; ++i) { + RandomDatum k1 = new RandomDatum(); + RandomDatum v1 = new RandomDatum(); + k1.readFields(originalIn); + v1.readFields(originalIn); + + RandomDatum k2 = new RandomDatum(); + RandomDatum v2 = new RandomDatum(); + k2.readFields(dataIn); + v2.readFields(dataIn); + assertTrue("original and encrypted-then-decrypted-output not equal", + k1.equals(k2) && v1.equals(v2)); + + // original and encrypted-then-decrypted-output have the same hashCode + Map m = new HashMap(); + m.put(k1, k1.toString()); + m.put(v1, v1.toString()); + String result = m.get(k2); + assertEquals("k1 and k2 hashcode not equal", result, k1.toString()); + result = m.get(v2); + assertEquals("v1 and v2 hashcode not equal", result, v1.toString()); + } + + // Decrypt data byte-at-a-time + originalData.reset(data.getData(), 0, data.getLength()); + decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, + encryptedDataBuffer.getLength()); + in = new CryptoInputStream(decryptedDataBuffer, + codec, bufferSize, key, iv); + + // Check + originalIn = new DataInputStream(new BufferedInputStream(originalData)); + int expected; + do { + expected = originalIn.read(); + assertEquals("Decrypted stream read by byte does not match", + expected, in.read()); + } while (expected != -1); + + LOG.info("SUCCESS! Completed checking " + count + " records"); + + // Check secure random generator + testSecureRandom(codec); + } + + /** Test secure random generator */ + private void testSecureRandom(CryptoCodec codec) { + // len = 16 + checkSecureRandom(codec, 16); + // len = 32 + checkSecureRandom(codec, 32); + // len = 128 + checkSecureRandom(codec, 128); + } + + private void checkSecureRandom(CryptoCodec codec, int len) { byte[] rand = new byte[len]; byte[] rand1 = new byte[len]; codec.generateSecureRandom(rand); @@ -56,28 +183,6 @@ private void checkSecureRandom(int len) { Assert.assertEquals(len, rand.length); Assert.assertEquals(len, rand1.length); - Assert.assertFalse(bytesArrayEquals(rand, rand1)); - } - - private boolean bytesArrayEquals(byte[] expected, byte[] actual) { - if ((expected == null && actual != null) || - (expected != null && actual == null)) { - return false; - } - if (expected == null && actual == null) { - return true; - } - - if (expected.length != actual.length) { - return false; - } - - for (int i = 0; i < expected.length; i++) { - if (expected[i] != actual[i]) { - return false; - } - } - - return true; + Assert.assertFalse(Arrays.equals(rand, rand1)); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java index 9efaca95c8..473c17738e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; import org.apache.hadoop.io.compress.SnappyCodec; import org.apache.hadoop.io.compress.zlib.ZlibFactory; @@ -54,6 +55,9 @@ public void testNativeCodeLoaded() { if (NativeCodeLoader.buildSupportsSnappy()) { assertFalse(SnappyCodec.getLibraryName().isEmpty()); } + if (NativeCodeLoader.buildSupportsOpenssl()) { + assertFalse(OpensslCipher.getLibraryName().isEmpty()); + } assertFalse(Lz4Codec.getLibraryName().isEmpty()); LOG.info("TestNativeCodeLoader: libhadoop.so is loaded."); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt index f6f68e433c..7c057bb8bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt @@ -10,6 +10,9 @@ fs-encryption (Unreleased) IMPROVEMENTS + HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL + (hitliuyi via cmccabe) + HDFS-6387. HDFS CLI admin tool for creating & deleting an encryption zone. (clamb) diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index dcb3875a12..e6dc75d3ca 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -41,6 +41,8 @@ UNDEF false false + false + false @@ -349,6 +351,10 @@ cd "${snappy.lib}" $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR) fi + if [ "${bundle.openssl}" = "true" ] ; then + cd "${openssl.lib}" + $$TAR *crypto* | (cd $${TARGET_DIR}/; $$UNTAR) + fi fi BIN_DIR="${BUILD_DIR}/bin" if [ -d $${BIN_DIR} ] ; then @@ -362,6 +368,12 @@ $$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR) fi fi + if [ "${bundle.openssl.in.bin}" = "true" ] ; then + if [ "${bundle.openssl}" = "true" ] ; then + cd "${openssl.lib}" + $$TAR *crypto* | (cd $${TARGET_BIN_DIR}/; $$UNTAR) + fi + fi fi diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 472febf87a..ed20742c32 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1031,6 +1031,7 @@ file:/dev/urandom true + true @@ -1041,6 +1042,7 @@ ${env.PATH};${hadoop.common.build.dir}/bin;${snappy.lib} + ${env.PATH};${hadoop.common.build.dir}/bin;${openssl.lib}