HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL (hitliuyi via cmccabe)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1607768 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe
parent
2a3bccddd9
commit
bfef9807a0
21
BUILDING.txt
21
BUILDING.txt
@ -81,6 +81,27 @@ Maven build goals:
|
||||
the final tar file. This option requires that -Dsnappy.lib is also given,
|
||||
and it ignores the -Dsnappy.prefix option.
|
||||
|
||||
OpenSSL build options:
|
||||
|
||||
OpenSSL includes a crypto library that can be utilized by the native code.
|
||||
It is currently an optional component, meaning that Hadoop can be built with
|
||||
or without this dependency.
|
||||
|
||||
* Use -Drequire.openssl to fail the build if libcrypto.so is not found.
|
||||
If this option is not specified and the openssl library is missing,
|
||||
we silently build a version of libhadoop.so that cannot make use of
|
||||
openssl. This option is recommended if you plan on making use of openssl
|
||||
and want to get more repeatable builds.
|
||||
* Use -Dopenssl.prefix to specify a nonstandard location for the libcrypto
|
||||
header files and library files. You do not need this option if you have
|
||||
installed openssl using a package manager.
|
||||
* Use -Dopenssl.lib to specify a nonstandard location for the libcrypto library
|
||||
files. Similarly to openssl.prefix, you do not need this option if you have
|
||||
installed openssl using a package manager.
|
||||
* Use -Dbundle.openssl to copy the contents of the openssl.lib directory into
|
||||
the final tar file. This option requires that -Dopenssl.lib is also given,
|
||||
and it ignores the -Dopenssl.prefix option.
|
||||
|
||||
Tests options:
|
||||
|
||||
* Use -DskipTests to skip tests when running the following Maven goals:
|
||||
|
||||
@ -483,6 +483,10 @@
|
||||
<snappy.lib></snappy.lib>
|
||||
<snappy.include></snappy.include>
|
||||
<require.snappy>false</require.snappy>
|
||||
<openssl.prefix></openssl.prefix>
|
||||
<openssl.lib></openssl.lib>
|
||||
<openssl.include></openssl.include>
|
||||
<require.openssl>false</require.openssl>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
@ -532,6 +536,7 @@
|
||||
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
|
||||
@ -552,7 +557,7 @@
|
||||
<configuration>
|
||||
<target>
|
||||
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
|
||||
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
|
||||
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
|
||||
</exec>
|
||||
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
|
||||
<arg line="VERBOSE=1"/>
|
||||
@ -596,6 +601,11 @@
|
||||
<snappy.include></snappy.include>
|
||||
<require.snappy>false</require.snappy>
|
||||
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
|
||||
<openssl.prefix></openssl.prefix>
|
||||
<openssl.lib></openssl.lib>
|
||||
<openssl.include></openssl.include>
|
||||
<require.openssl>false</require.openssl>
|
||||
<bundle.openssl.in.bin>true</bundle.openssl.in.bin>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
@ -641,6 +651,7 @@
|
||||
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
|
||||
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
|
||||
</javahClassNames>
|
||||
<javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
|
||||
@ -685,6 +696,10 @@
|
||||
<argument>/p:CustomSnappyLib=${snappy.lib}</argument>
|
||||
<argument>/p:CustomSnappyInclude=${snappy.include}</argument>
|
||||
<argument>/p:RequireSnappy=${require.snappy}</argument>
|
||||
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
|
||||
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
|
||||
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
|
||||
<argument>/p:RequireOpenssl=${require.openssl}</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
|
||||
@ -145,6 +145,37 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
|
||||
ENDIF(REQUIRE_SNAPPY)
|
||||
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
|
||||
|
||||
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
|
||||
set_find_shared_library_version("1.0.0")
|
||||
SET(OPENSSL_NAME "crypto")
|
||||
IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
|
||||
SET(OPENSSL_NAME "eay32")
|
||||
ENDIF()
|
||||
find_library(OPENSSL_LIBRARY
|
||||
NAMES ${OPENSSL_NAME}
|
||||
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib
|
||||
${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH)
|
||||
find_library(OPENSSL_LIBRARY
|
||||
NAMES ${OPENSSL_NAME})
|
||||
SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
|
||||
find_path(OPENSSL_INCLUDE_DIR
|
||||
NAMES openssl/evp.h
|
||||
PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include
|
||||
${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH)
|
||||
find_path(OPENSSL_INCLUDE_DIR
|
||||
NAMES openssl/evp.h)
|
||||
if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||
GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
|
||||
SET(OPENSSL_SOURCE_FILES
|
||||
"${D}/crypto/OpensslCipher.c")
|
||||
else (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||
SET(OPENSSL_INCLUDE_DIR "")
|
||||
SET(OPENSSL_SOURCE_FILES "")
|
||||
IF(REQUIRE_OPENSSL)
|
||||
MESSAGE(FATAL_ERROR "Required openssl library could not be found. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_INCLUDE_DIR=${CUSTOM_OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
|
||||
ENDIF(REQUIRE_OPENSSL)
|
||||
endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||
|
||||
include_directories(
|
||||
${GENERATED_JAVAH}
|
||||
main/native/src
|
||||
@ -155,6 +186,7 @@ include_directories(
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
${BZIP2_INCLUDE_DIR}
|
||||
${SNAPPY_INCLUDE_DIR}
|
||||
${OPENSSL_INCLUDE_DIR}
|
||||
${D}/util
|
||||
)
|
||||
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
|
||||
@ -172,6 +204,7 @@ add_dual_library(hadoop
|
||||
${D}/io/compress/lz4/lz4.c
|
||||
${D}/io/compress/lz4/lz4hc.c
|
||||
${SNAPPY_SOURCE_FILES}
|
||||
${OPENSSL_SOURCE_FILES}
|
||||
${D}/io/compress/zlib/ZlibCompressor.c
|
||||
${D}/io/compress/zlib/ZlibDecompressor.c
|
||||
${BZIP2_SOURCE_FILES}
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
|
||||
#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
|
||||
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
|
||||
#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
|
||||
#cmakedefine HAVE_SYNC_FILE_RANGE
|
||||
#cmakedefine HAVE_POSIX_FADVISE
|
||||
|
||||
|
||||
@ -1,71 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.crypto;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Evolving
|
||||
public abstract class AESCTRCryptoCodec extends CryptoCodec {
|
||||
|
||||
protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
|
||||
|
||||
/**
|
||||
* For AES, the algorithm block is fixed size of 128 bits.
|
||||
* @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
|
||||
*/
|
||||
private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize();
|
||||
private static final int CTR_OFFSET = 8;
|
||||
|
||||
@Override
|
||||
public CipherSuite getCipherSuite() {
|
||||
return SUITE;
|
||||
}
|
||||
|
||||
/**
|
||||
* The IV is produced by adding the initial IV to the counter. IV length
|
||||
* should be the same as {@link #AES_BLOCK_SIZE}
|
||||
*/
|
||||
@Override
|
||||
public void calculateIV(byte[] initIV, long counter, byte[] IV) {
|
||||
Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE);
|
||||
Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE);
|
||||
|
||||
System.arraycopy(initIV, 0, IV, 0, CTR_OFFSET);
|
||||
long l = (initIV[CTR_OFFSET + 0] << 56)
|
||||
+ ((initIV[CTR_OFFSET + 1] & 0xFF) << 48)
|
||||
+ ((initIV[CTR_OFFSET + 2] & 0xFF) << 40)
|
||||
+ ((initIV[CTR_OFFSET + 3] & 0xFF) << 32)
|
||||
+ ((initIV[CTR_OFFSET + 4] & 0xFF) << 24)
|
||||
+ ((initIV[CTR_OFFSET + 5] & 0xFF) << 16)
|
||||
+ ((initIV[CTR_OFFSET + 6] & 0xFF) << 8)
|
||||
+ (initIV[CTR_OFFSET + 7] & 0xFF);
|
||||
l += counter;
|
||||
IV[CTR_OFFSET + 0] = (byte) (l >>> 56);
|
||||
IV[CTR_OFFSET + 1] = (byte) (l >>> 48);
|
||||
IV[CTR_OFFSET + 2] = (byte) (l >>> 40);
|
||||
IV[CTR_OFFSET + 3] = (byte) (l >>> 32);
|
||||
IV[CTR_OFFSET + 4] = (byte) (l >>> 24);
|
||||
IV[CTR_OFFSET + 5] = (byte) (l >>> 16);
|
||||
IV[CTR_OFFSET + 6] = (byte) (l >>> 8);
|
||||
IV[CTR_OFFSET + 7] = (byte) (l);
|
||||
}
|
||||
}
|
||||
@ -35,7 +35,7 @@ public abstract class CryptoCodec implements Configurable {
|
||||
|
||||
public static CryptoCodec getInstance(Configuration conf) {
|
||||
final Class<? extends CryptoCodec> klass = conf.getClass(
|
||||
HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JCEAESCTRCryptoCodec.class,
|
||||
HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class,
|
||||
CryptoCodec.class);
|
||||
return ReflectionUtils.newInstance(klass, conf);
|
||||
}
|
||||
|
||||
@ -1,159 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.crypto;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.SecureRandom;
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.spec.IvParameterSpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY;
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT;
|
||||
|
||||
/**
|
||||
* Implement the AES-CTR crypto codec using JCE provider.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class JCEAESCTRCryptoCodec extends AESCTRCryptoCodec {
|
||||
private Configuration conf;
|
||||
private String provider;
|
||||
private SecureRandom random;
|
||||
|
||||
public JCEAESCTRCryptoCodec() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return conf;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
this.conf = conf;
|
||||
provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
|
||||
final String secureRandomAlg = conf.get(
|
||||
HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY,
|
||||
HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT);
|
||||
try {
|
||||
random = (provider != null) ?
|
||||
SecureRandom.getInstance(secureRandomAlg, provider) :
|
||||
SecureRandom.getInstance(secureRandomAlg);
|
||||
} catch (GeneralSecurityException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Encryptor createEncryptor() throws GeneralSecurityException {
|
||||
return new JCEAESCTRCipher(Cipher.ENCRYPT_MODE, provider);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decryptor createDecryptor() throws GeneralSecurityException {
|
||||
return new JCEAESCTRCipher(Cipher.DECRYPT_MODE, provider);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void generateSecureRandom(byte[] bytes) {
|
||||
random.nextBytes(bytes);
|
||||
}
|
||||
|
||||
private static class JCEAESCTRCipher implements Encryptor, Decryptor {
|
||||
private final Cipher cipher;
|
||||
private final int mode;
|
||||
private boolean contextReset = false;
|
||||
|
||||
public JCEAESCTRCipher(int mode, String provider)
|
||||
throws GeneralSecurityException {
|
||||
this.mode = mode;
|
||||
if (provider == null || provider.isEmpty()) {
|
||||
cipher = Cipher.getInstance(SUITE.getName());
|
||||
} else {
|
||||
cipher = Cipher.getInstance(SUITE.getName(), provider);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(byte[] key, byte[] iv) throws IOException {
|
||||
Preconditions.checkNotNull(key);
|
||||
Preconditions.checkNotNull(iv);
|
||||
contextReset = false;
|
||||
try {
|
||||
cipher.init(mode, new SecretKeySpec(key, "AES"),
|
||||
new IvParameterSpec(iv));
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AES-CTR will consume all of the input data. It requires enough space in
|
||||
* the destination buffer to encrypt entire input buffer.
|
||||
*/
|
||||
@Override
|
||||
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||
throws IOException {
|
||||
process(inBuffer, outBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* AES-CTR will consume all of the input data. It requires enough space in
|
||||
* the destination buffer to decrypt entire input buffer.
|
||||
*/
|
||||
@Override
|
||||
public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||
throws IOException {
|
||||
process(inBuffer, outBuffer);
|
||||
}
|
||||
|
||||
private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
|
||||
throws IOException {
|
||||
try {
|
||||
int inputSize = inBuffer.remaining();
|
||||
// Cipher#update will maintain crypto context.
|
||||
int n = cipher.update(inBuffer, outBuffer);
|
||||
if (n < inputSize) {
|
||||
/**
|
||||
* Typically code will not get here. Cipher#update will consume all
|
||||
* input data and put result in outBuffer.
|
||||
* Cipher#doFinal will reset the crypto context.
|
||||
*/
|
||||
contextReset = true;
|
||||
cipher.doFinal(inBuffer, outBuffer);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isContextReset() {
|
||||
return contextReset;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -78,6 +78,11 @@ public static boolean isNativeCodeLoaded() {
|
||||
* Returns true only if this build was compiled with support for snappy.
|
||||
*/
|
||||
public static native boolean buildSupportsSnappy();
|
||||
|
||||
/**
|
||||
* Returns true only if this build was compiled with support for openssl.
|
||||
*/
|
||||
public static native boolean buildSupportsOpenssl();
|
||||
|
||||
public static native String getLibraryName();
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.crypto.OpensslCipher;
|
||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
|
||||
@ -57,12 +58,14 @@ public static void main(String[] args) {
|
||||
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
|
||||
boolean zlibLoaded = false;
|
||||
boolean snappyLoaded = false;
|
||||
boolean opensslLoaded = false;
|
||||
// lz4 is linked within libhadoop
|
||||
boolean lz4Loaded = nativeHadoopLoaded;
|
||||
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
||||
String hadoopLibraryName = "";
|
||||
String zlibLibraryName = "";
|
||||
String snappyLibraryName = "";
|
||||
String opensslLibraryName = "";
|
||||
String lz4LibraryName = "";
|
||||
String bzip2LibraryName = "";
|
||||
if (nativeHadoopLoaded) {
|
||||
@ -76,6 +79,11 @@ public static void main(String[] args) {
|
||||
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
|
||||
snappyLibraryName = SnappyCodec.getLibraryName();
|
||||
}
|
||||
opensslLoaded = NativeCodeLoader.buildSupportsOpenssl() &&
|
||||
OpensslCipher.isNativeCodeLoaded();
|
||||
if (opensslLoaded) {
|
||||
opensslLibraryName = OpensslCipher.getLibraryName();
|
||||
}
|
||||
if (lz4Loaded) {
|
||||
lz4LibraryName = Lz4Codec.getLibraryName();
|
||||
}
|
||||
@ -84,11 +92,12 @@ public static void main(String[] args) {
|
||||
}
|
||||
}
|
||||
System.out.println("Native library checking:");
|
||||
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
||||
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
||||
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
|
||||
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
||||
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
||||
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
||||
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
||||
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
|
||||
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
||||
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
||||
System.out.printf("openssl: %b %s\n", opensslLoaded, opensslLibraryName);
|
||||
if ((!nativeHadoopLoaded) ||
|
||||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
||||
// return 1 to indicated check failed
|
||||
|
||||
@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
|
||||
#endif
|
||||
}
|
||||
|
||||
JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
|
||||
(JNIEnv *env, jclass clazz)
|
||||
{
|
||||
#ifdef HADOOP_OPENSSL_LIBRARY
|
||||
return JNI_TRUE;
|
||||
#else
|
||||
return JNI_FALSE;
|
||||
#endif
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
|
||||
(JNIEnv *env, jclass clazz)
|
||||
{
|
||||
|
||||
@ -17,38 +17,165 @@
|
||||
*/
|
||||
package org.apache.hadoop.crypto;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.IOException;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.junit.AfterClass;
|
||||
import org.apache.hadoop.io.DataInputBuffer;
|
||||
import org.apache.hadoop.io.DataOutputBuffer;
|
||||
import org.apache.hadoop.io.RandomDatum;
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
import org.apache.hadoop.util.ReflectionUtils;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestCryptoCodec {
|
||||
private static CryptoCodec codec;
|
||||
private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class);
|
||||
private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||
0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
|
||||
private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
|
||||
0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
|
||||
private static final int bufferSize = 4096;
|
||||
|
||||
@BeforeClass
|
||||
public static void init() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
codec = CryptoCodec.getInstance(conf);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void shutdown() throws Exception {
|
||||
}
|
||||
private Configuration conf = new Configuration();
|
||||
private int count = 10000;
|
||||
private int seed = new Random().nextInt();
|
||||
|
||||
@Test(timeout=120000)
|
||||
public void testSecureRandom() throws Exception {
|
||||
// len = 16
|
||||
checkSecureRandom(16);
|
||||
|
||||
// len = 32
|
||||
checkSecureRandom(32);
|
||||
|
||||
// len = 128
|
||||
checkSecureRandom(128);
|
||||
public void testJceAesCtrCryptoCodec() throws Exception {
|
||||
cryptoCodecTest(conf, seed, 0,
|
||||
"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
|
||||
cryptoCodecTest(conf, seed, count,
|
||||
"org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
|
||||
}
|
||||
|
||||
private void checkSecureRandom(int len) {
|
||||
@Test(timeout=1200000)
|
||||
public void testOpensslAesCtrCryptoCodec() throws Exception {
|
||||
if (NativeCodeLoader.buildSupportsOpenssl()) {
|
||||
Assert.assertTrue(OpensslCipher.isNativeCodeLoaded());
|
||||
}
|
||||
if (OpensslCipher.isNativeCodeLoaded()) {
|
||||
cryptoCodecTest(conf, seed, 0,
|
||||
"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
|
||||
cryptoCodecTest(conf, seed, count,
|
||||
"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
|
||||
}
|
||||
}
|
||||
|
||||
private void cryptoCodecTest(Configuration conf, int seed, int count,
|
||||
String codecClass) throws IOException, GeneralSecurityException {
|
||||
CryptoCodec codec = null;
|
||||
try {
|
||||
codec = (CryptoCodec)ReflectionUtils.newInstance(
|
||||
conf.getClassByName(codecClass), conf);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
throw new IOException("Illegal crypto codec!");
|
||||
}
|
||||
LOG.info("Created a Codec object of type: " + codecClass);
|
||||
|
||||
// Generate data
|
||||
DataOutputBuffer data = new DataOutputBuffer();
|
||||
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
|
||||
for(int i = 0; i < count; ++i) {
|
||||
generator.next();
|
||||
RandomDatum key = generator.getKey();
|
||||
RandomDatum value = generator.getValue();
|
||||
|
||||
key.write(data);
|
||||
value.write(data);
|
||||
}
|
||||
LOG.info("Generated " + count + " records");
|
||||
|
||||
// Encrypt data
|
||||
DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer();
|
||||
CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer,
|
||||
codec, bufferSize, key, iv);
|
||||
out.write(data.getData(), 0, data.getLength());
|
||||
out.flush();
|
||||
out.close();
|
||||
LOG.info("Finished encrypting data");
|
||||
|
||||
// Decrypt data
|
||||
DataInputBuffer decryptedDataBuffer = new DataInputBuffer();
|
||||
decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
|
||||
encryptedDataBuffer.getLength());
|
||||
CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer,
|
||||
codec, bufferSize, key, iv);
|
||||
DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in));
|
||||
|
||||
// Check
|
||||
DataInputBuffer originalData = new DataInputBuffer();
|
||||
originalData.reset(data.getData(), 0, data.getLength());
|
||||
DataInputStream originalIn = new DataInputStream(
|
||||
new BufferedInputStream(originalData));
|
||||
|
||||
for(int i=0; i < count; ++i) {
|
||||
RandomDatum k1 = new RandomDatum();
|
||||
RandomDatum v1 = new RandomDatum();
|
||||
k1.readFields(originalIn);
|
||||
v1.readFields(originalIn);
|
||||
|
||||
RandomDatum k2 = new RandomDatum();
|
||||
RandomDatum v2 = new RandomDatum();
|
||||
k2.readFields(dataIn);
|
||||
v2.readFields(dataIn);
|
||||
assertTrue("original and encrypted-then-decrypted-output not equal",
|
||||
k1.equals(k2) && v1.equals(v2));
|
||||
|
||||
// original and encrypted-then-decrypted-output have the same hashCode
|
||||
Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
|
||||
m.put(k1, k1.toString());
|
||||
m.put(v1, v1.toString());
|
||||
String result = m.get(k2);
|
||||
assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
|
||||
result = m.get(v2);
|
||||
assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
|
||||
}
|
||||
|
||||
// Decrypt data byte-at-a-time
|
||||
originalData.reset(data.getData(), 0, data.getLength());
|
||||
decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
|
||||
encryptedDataBuffer.getLength());
|
||||
in = new CryptoInputStream(decryptedDataBuffer,
|
||||
codec, bufferSize, key, iv);
|
||||
|
||||
// Check
|
||||
originalIn = new DataInputStream(new BufferedInputStream(originalData));
|
||||
int expected;
|
||||
do {
|
||||
expected = originalIn.read();
|
||||
assertEquals("Decrypted stream read by byte does not match",
|
||||
expected, in.read());
|
||||
} while (expected != -1);
|
||||
|
||||
LOG.info("SUCCESS! Completed checking " + count + " records");
|
||||
|
||||
// Check secure random generator
|
||||
testSecureRandom(codec);
|
||||
}
|
||||
|
||||
/** Test secure random generator */
|
||||
private void testSecureRandom(CryptoCodec codec) {
|
||||
// len = 16
|
||||
checkSecureRandom(codec, 16);
|
||||
// len = 32
|
||||
checkSecureRandom(codec, 32);
|
||||
// len = 128
|
||||
checkSecureRandom(codec, 128);
|
||||
}
|
||||
|
||||
private void checkSecureRandom(CryptoCodec codec, int len) {
|
||||
byte[] rand = new byte[len];
|
||||
byte[] rand1 = new byte[len];
|
||||
codec.generateSecureRandom(rand);
|
||||
@ -56,28 +183,6 @@ private void checkSecureRandom(int len) {
|
||||
|
||||
Assert.assertEquals(len, rand.length);
|
||||
Assert.assertEquals(len, rand1.length);
|
||||
Assert.assertFalse(bytesArrayEquals(rand, rand1));
|
||||
}
|
||||
|
||||
private boolean bytesArrayEquals(byte[] expected, byte[] actual) {
|
||||
if ((expected == null && actual != null) ||
|
||||
(expected != null && actual == null)) {
|
||||
return false;
|
||||
}
|
||||
if (expected == null && actual == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (expected.length != actual.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int i = 0; i < expected.length; i++) {
|
||||
if (expected[i] != actual[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
Assert.assertFalse(Arrays.equals(rand, rand1));
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.crypto.OpensslCipher;
|
||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
|
||||
@ -54,6 +55,9 @@ public void testNativeCodeLoaded() {
|
||||
if (NativeCodeLoader.buildSupportsSnappy()) {
|
||||
assertFalse(SnappyCodec.getLibraryName().isEmpty());
|
||||
}
|
||||
if (NativeCodeLoader.buildSupportsOpenssl()) {
|
||||
assertFalse(OpensslCipher.getLibraryName().isEmpty());
|
||||
}
|
||||
assertFalse(Lz4Codec.getLibraryName().isEmpty());
|
||||
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
|
||||
}
|
||||
|
||||
@ -10,6 +10,9 @@ fs-encryption (Unreleased)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL
|
||||
(hitliuyi via cmccabe)
|
||||
|
||||
HDFS-6387. HDFS CLI admin tool for creating & deleting an
|
||||
encryption zone. (clamb)
|
||||
|
||||
|
||||
@ -41,6 +41,8 @@
|
||||
<hadoop.component>UNDEF</hadoop.component>
|
||||
<bundle.snappy>false</bundle.snappy>
|
||||
<bundle.snappy.in.bin>false</bundle.snappy.in.bin>
|
||||
<bundle.openssl>false</bundle.openssl>
|
||||
<bundle.openssl.in.bin>false</bundle.openssl.in.bin>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@ -349,6 +351,10 @@
|
||||
cd "${snappy.lib}"
|
||||
$$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
|
||||
fi
|
||||
if [ "${bundle.openssl}" = "true" ] ; then
|
||||
cd "${openssl.lib}"
|
||||
$$TAR *crypto* | (cd $${TARGET_DIR}/; $$UNTAR)
|
||||
fi
|
||||
fi
|
||||
BIN_DIR="${BUILD_DIR}/bin"
|
||||
if [ -d $${BIN_DIR} ] ; then
|
||||
@ -362,6 +368,12 @@
|
||||
$$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
|
||||
fi
|
||||
fi
|
||||
if [ "${bundle.openssl.in.bin}" = "true" ] ; then
|
||||
if [ "${bundle.openssl}" = "true" ] ; then
|
||||
cd "${openssl.lib}"
|
||||
$$TAR *crypto* | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
</echo>
|
||||
<exec executable="sh" dir="${project.build.directory}" failonerror="true">
|
||||
|
||||
@ -1031,6 +1031,7 @@
|
||||
<!-- attempt to open a file at this path. -->
|
||||
<java.security.egd>file:/dev/urandom</java.security.egd>
|
||||
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
|
||||
<bundle.openssl.in.bin>true</bundle.openssl.in.bin>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
@ -1041,6 +1042,7 @@
|
||||
<environmentVariables>
|
||||
<!-- Specify where to look for the native DLL on Windows -->
|
||||
<PATH>${env.PATH};${hadoop.common.build.dir}/bin;${snappy.lib}</PATH>
|
||||
<PATH>${env.PATH};${hadoop.common.build.dir}/bin;${openssl.lib}</PATH>
|
||||
</environmentVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user