big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14524. Make CryptoCodec Closeable so it can be cleaned up proactively.

Browse Source
This commit is contained in:
Xiao Chen 2017-06-15 14:25:52 -07:00
parent 942ecbbc98
commit 4ebc23ba7b
9 changed files with 82 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java
View File

@ -22,6 +22,8 @@
import com.google.common.base.Preconditions;
import java.io.IOException;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class AesCtrCryptoCodec extends CryptoCodec {
@ -61,4 +63,8 @@ public void calculateIV(byte[] initIV, long counter, byte[] IV) {
IV[i] = (byte) sum;
}
}
@Override
public void close() throws IOException {
}
}

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.crypto;
import java.io.Closeable;
import java.security.GeneralSecurityException;
import java.util.List;
@ -42,7 +43,7 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class CryptoCodec implements Configurable {
public abstract class CryptoCodec implements Configurable, Closeable {
public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class);
/**

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
View File

@ -315,6 +315,7 @@ public void close() throws IOException {
super.close();
freeBuffers();
codec.close();
closed = true;
}

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
View File

@ -239,6 +239,7 @@ public synchronized void close() throws IOException {
flush();
if (closeOutputStream) {
super.close();
codec.close();
}
freeBuffers();
} finally {

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
View File

@ -19,6 +19,7 @@
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
@ -89,7 +90,17 @@ public Decryptor createDecryptor() throws GeneralSecurityException {
public void generateSecureRandom(byte[] bytes) {
random.nextBytes(bytes);
}
@Override
public void close() throws IOException {
try {
Closeable r = (Closeable) this.random;
r.close();
} catch (ClassCastException e) {
}
super.close();
}
private static class OpensslAesCtrCipher implements Encryptor, Decryptor {
private final OpensslCipher cipher;
private final int mode;

27
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
View File

@ -274,12 +274,16 @@ public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
// Generate random bytes for new key and IV
CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
final byte[] newKey = new byte[encryptionKey.getMaterial().length];
cc.generateSecureRandom(newKey);
final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
cc.generateSecureRandom(iv);
Encryptor encryptor = cc.createEncryptor();
return generateEncryptedKey(encryptor, encryptionKey, newKey, iv);
try {
final byte[] newKey = new byte[encryptionKey.getMaterial().length];
cc.generateSecureRandom(newKey);
final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
cc.generateSecureRandom(iv);
Encryptor encryptor = cc.createEncryptor();
return generateEncryptedKey(encryptor, encryptionKey, newKey, iv);
} finally {
cc.close();
}
}
private EncryptedKeyVersion generateEncryptedKey(final Encryptor encryptor,
@ -322,9 +326,13 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
final KeyVersion dek = decryptEncryptedKey(ekv);
final CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
final Encryptor encryptor = cc.createEncryptor();
return generateEncryptedKey(encryptor, ekNow, dek.getMaterial(),
ekv.getEncryptedKeyIv());
try {
final Encryptor encryptor = cc.createEncryptor();
return generateEncryptedKey(encryptor, ekNow, dek.getMaterial(),
ekv.getEncryptedKeyIv());
} finally {
cc.close();
}
}
@Override
@ -364,6 +372,7 @@ public KeyVersion decryptEncryptedKey(
bbOut.flip();
byte[] decryptedKey = new byte[keyLen];
bbOut.get(decryptedKey);
cc.close();
return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
}

32
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
View File

@ -18,12 +18,17 @@
package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.random.OsSecureRandom;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
extends TestCryptoStreams {
@ -32,8 +37,7 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
public static void init() throws Exception {
GenericTestUtils.assumeInNativeProfile();
Configuration conf = new Configuration();
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
OpensslAesCtrCryptoCodec.class.getName());
codec = CryptoCodec.getInstance(conf);
assertNotNull("Unable to instantiate codec " +
@ -42,4 +46,28 @@ public static void init() throws Exception {
assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
codec.getClass().getCanonicalName());
}
@Test
public void testCodecClosesRandom() throws Exception {
GenericTestUtils.assumeInNativeProfile();
Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
OpensslAesCtrCryptoCodec.class.getName());
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY,
OsSecureRandom.class.getName());
CryptoCodec codecWithRandom = CryptoCodec.getInstance(conf);
assertNotNull(
"Unable to instantiate codec " + OpensslAesCtrCryptoCodec.class
.getName() + ", is the required " + "version of OpenSSL installed?",
codecWithRandom);
OsSecureRandom random =
(OsSecureRandom) Whitebox.getInternalState(codecWithRandom, "random");
// trigger the OsSecureRandom to create an internal FileInputStream
random.nextBytes(new byte[10]);
assertNotNull(Whitebox.getInternalState(random, "stream"));
// verify closing the codec closes the codec's random's stream.
codecWithRandom.close();
assertNull(Whitebox.getInternalState(random, "stream"));
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
View File

@ -286,6 +286,7 @@ public static CipherOption negotiateCipherOption(Configuration conf,
codec.generateSecureRandom(inIv);
codec.generateSecureRandom(outKey);
codec.generateSecureRandom(outIv);
codec.close();
return new CipherOption(suite, inKey, inIv, outKey, outIv);
}
}

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/CryptoUtils.java
View File

@ -66,16 +66,24 @@ public static byte[] createIV(Configuration conf) throws IOException {
if (isEncryptedSpillEnabled(conf)) {
byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()];
cryptoCodec.generateSecureRandom(iv);
cryptoCodec.close();
return iv;
} else {
return null;
}
}
public static int cryptoPadding(Configuration conf) {
public static int cryptoPadding(Configuration conf) throws IOException {
// Sizeof(IV) + long(start-offset)
return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf)
.getCipherSuite().getAlgorithmBlockSize() + 8 : 0;
if (!isEncryptedSpillEnabled(conf)) {
return 0;
}
final CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf);
try {
return cryptoCodec.getCipherSuite().getAlgorithmBlockSize() + 8;
} finally {
cryptoCodec.close();
}
}
private static byte[] getEncryptionKey() throws IOException {