diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index bb9e1e4ab5..18758515b4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -399,6 +399,9 @@ Trunk (Unreleased) HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is thread-unsafe. (benoyantony viat tucu) + HADOOP-10881. Clarify usage of encryption and encrypted encryption + key in KeyProviderCryptoExtension. (wang) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java index e4b822d2c6..0ba73f1519 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.security.GeneralSecurityException; import java.security.SecureRandom; - import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; @@ -30,51 +29,109 @@ import org.apache.hadoop.classification.InterfaceAudience; /** - * A KeyProvider with Cytographic Extensions specifically for generating - * Encrypted Keys as well as decrypting them + * A KeyProvider with Cryptographic Extensions specifically for generating + * and decrypting encrypted encryption keys. * */ @InterfaceAudience.Private public class KeyProviderCryptoExtension extends KeyProviderExtension { + /** + * Designates an encrypted encryption key, or EEK. + */ public static final String EEK = "EEK"; + /** + * Designates a decrypted encrypted encryption key, that is, an encryption key + * (EK). + */ public static final String EK = "EK"; /** - * This is a holder class whose instance contains the keyVersionName, iv - * used to generate the encrypted Key and the encrypted KeyVersion + * An encrypted encryption key (EEK) and related information. An EEK must be + * decrypted using the key's encryption key before it can be used. */ public static class EncryptedKeyVersion { - private String keyName; - private String keyVersionName; - private byte[] iv; - private KeyVersion encryptedKey; + private String encryptionKeyName; + private String encryptionKeyVersionName; + private byte[] encryptedKeyIv; + private KeyVersion encryptedKeyVersion; - protected EncryptedKeyVersion(String keyName, String keyVersionName, - byte[] iv, KeyVersion encryptedKey) { - this.keyName = keyName; - this.keyVersionName = keyVersionName; - this.iv = iv; - this.encryptedKey = encryptedKey; + /** + * Create a new EncryptedKeyVersion. + * + * @param keyName Name of the encryption key used to + * encrypt the encrypted key. + * @param encryptionKeyVersionName Version name of the encryption key used + * to encrypt the encrypted key. + * @param encryptedKeyIv Initialization vector of the encrypted + * key. The IV of the encryption key used to + * encrypt the encrypted key is derived from + * this IV. + * @param encryptedKeyVersion The encrypted encryption key version. + */ + protected EncryptedKeyVersion(String keyName, + String encryptionKeyVersionName, byte[] encryptedKeyIv, + KeyVersion encryptedKeyVersion) { + this.encryptionKeyName = keyName; + this.encryptionKeyVersionName = encryptionKeyVersionName; + this.encryptedKeyIv = encryptedKeyIv; + this.encryptedKeyVersion = encryptedKeyVersion; } - public String getKeyName() { - return keyName; + /** + * @return Name of the encryption key used to encrypt the encrypted key. + */ + public String getEncryptionKeyName() { + return encryptionKeyName; } - public String getKeyVersionName() { - return keyVersionName; + /** + * @return Version name of the encryption key used to encrypt the encrypted + * key. + */ + public String getEncryptionKeyVersionName() { + return encryptionKeyVersionName; } - public byte[] getIv() { - return iv; + /** + * @return Initialization vector of the encrypted key. The IV of the + * encryption key used to encrypt the encrypted key is derived from this + * IV. + */ + public byte[] getEncryptedKeyIv() { + return encryptedKeyIv; } - public KeyVersion getEncryptedKey() { - return encryptedKey; + /** + * @return The encrypted encryption key version. + */ + public KeyVersion getEncryptedKeyVersion() { + return encryptedKeyVersion; } + /** + * Derive the initialization vector (IV) for the encryption key from the IV + * of the encrypted key. This derived IV is used with the encryption key to + * decrypt the encrypted key. + *

+ * The alternative to this is using the same IV for both the encryption key + * and the encrypted key. Even a simple symmetric transformation like this + * improves security by avoiding IV re-use. IVs will also be fairly unique + * among different EEKs. + * + * @param encryptedKeyIV of the encrypted key (i.e. {@link + * #getEncryptedKeyIv()}) + * @return IV for the encryption key + */ + protected static byte[] deriveIV(byte[] encryptedKeyIV) { + byte[] rIv = new byte[encryptedKeyIV.length]; + // Do a simple XOR transformation to flip all the bits + for (int i = 0; i < encryptedKeyIV.length; i++) { + rIv[i] = (byte) (encryptedKeyIV[i] ^ 0xff); + } + return rIv; + } } /** @@ -141,53 +198,56 @@ private DefaultCryptoExtension(KeyProvider keyProvider) { this.keyProvider = keyProvider; } - // the IV used to encrypt a EK typically will be the same IV used to - // encrypt data with the EK. To avoid any chance of weakening the - // encryption because the same IV is used, we simply XOR the IV thus we - // are not using the same IV for 2 different encryptions (even if they - // are done using different keys) - private byte[] flipIV(byte[] iv) { - byte[] rIv = new byte[iv.length]; - for (int i = 0; i < iv.length; i++) { - rIv[i] = (byte) (iv[i] ^ 0xff); - } - return rIv; - } - @Override public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName) throws IOException, GeneralSecurityException { - KeyVersion keyVer = keyProvider.getCurrentKey(encryptionKeyName); - Preconditions.checkNotNull(keyVer, "No KeyVersion exists for key '%s' ", - encryptionKeyName); - byte[] newKey = new byte[keyVer.getMaterial().length]; - SecureRandom.getInstance("SHA1PRNG").nextBytes(newKey); + // Fetch the encryption key + KeyVersion encryptionKey = keyProvider.getCurrentKey(encryptionKeyName); + Preconditions.checkNotNull(encryptionKey, + "No KeyVersion exists for key '%s' ", encryptionKeyName); + // Generate random bytes for new key and IV Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); - byte[] iv = SecureRandom.getSeed(cipher.getBlockSize()); - cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(keyVer.getMaterial(), - "AES"), new IvParameterSpec(flipIV(iv))); - byte[] ek = cipher.doFinal(newKey); + SecureRandom random = SecureRandom.getInstance("SHA1PRNG"); + final byte[] newKey = new byte[encryptionKey.getMaterial().length]; + random.nextBytes(newKey); + final byte[] iv = random.generateSeed(cipher.getBlockSize()); + // Encryption key IV is derived from new key's IV + final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv); + // Encrypt the new key + cipher.init(Cipher.ENCRYPT_MODE, + new SecretKeySpec(encryptionKey.getMaterial(), "AES"), + new IvParameterSpec(encryptionIV)); + final byte[] encryptedKey = cipher.doFinal(newKey); return new EncryptedKeyVersion(encryptionKeyName, - keyVer.getVersionName(), iv, - new KeyVersion(keyVer.getName(), EEK, ek)); + encryptionKey.getVersionName(), iv, + new KeyVersion(encryptionKey.getName(), EEK, encryptedKey)); } @Override public KeyVersion decryptEncryptedKey( EncryptedKeyVersion encryptedKeyVersion) throws IOException, GeneralSecurityException { - KeyVersion keyVer = - keyProvider.getKeyVersion(encryptedKeyVersion.getKeyVersionName()); - Preconditions.checkNotNull(keyVer, "KeyVersion name '%s' does not exist", - encryptedKeyVersion.getKeyVersionName()); - KeyVersion keyVersion = encryptedKeyVersion.getEncryptedKey(); + // Fetch the encryption key material + final String encryptionKeyVersionName = + encryptedKeyVersion.getEncryptionKeyVersionName(); + final KeyVersion encryptionKey = + keyProvider.getKeyVersion(encryptionKeyVersionName); + Preconditions.checkNotNull(encryptionKey, + "KeyVersion name '%s' does not exist", encryptionKeyVersionName); + final byte[] encryptionKeyMaterial = encryptionKey.getMaterial(); + // Encryption key IV is determined from encrypted key's IV + final byte[] encryptionIV = + EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv()); + // Init the cipher with encryption key parameters Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); cipher.init(Cipher.DECRYPT_MODE, - new SecretKeySpec(keyVersion.getMaterial(), "AES"), - new IvParameterSpec(flipIV(encryptedKeyVersion.getIv()))); - byte[] ek = - cipher.doFinal(encryptedKeyVersion.getEncryptedKey().getMaterial()); - return new KeyVersion(keyVer.getName(), EK, ek); + new SecretKeySpec(encryptionKeyMaterial, "AES"), + new IvParameterSpec(encryptionIV)); + // Decrypt the encrypted key + final KeyVersion encryptedKV = + encryptedKeyVersion.getEncryptedKeyVersion(); + final byte[] decryptedKey = cipher.doFinal(encryptedKV.getMaterial()); + return new KeyVersion(encryptionKey.getName(), EK, decryptedKey); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 808b1bb102..06521a4359 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -646,25 +646,28 @@ public EncryptedKeyVersion generateEncryptedKey( public KeyVersion decryptEncryptedKey( EncryptedKeyVersion encryptedKeyVersion) throws IOException, GeneralSecurityException { - checkNotNull(encryptedKeyVersion.getKeyVersionName(), "versionName"); - checkNotNull(encryptedKeyVersion.getIv(), "iv"); - Preconditions.checkArgument(encryptedKeyVersion.getEncryptedKey() - .getVersionName().equals(KeyProviderCryptoExtension.EEK), + checkNotNull(encryptedKeyVersion.getEncryptionKeyVersionName(), + "versionName"); + checkNotNull(encryptedKeyVersion.getEncryptedKeyIv(), "iv"); + Preconditions.checkArgument( + encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() + .equals(KeyProviderCryptoExtension.EEK), "encryptedKey version name must be '%s', is '%s'", - KeyProviderCryptoExtension.EK, encryptedKeyVersion.getEncryptedKey() - .getVersionName()); - checkNotNull(encryptedKeyVersion.getEncryptedKey(), "encryptedKey"); + KeyProviderCryptoExtension.EK, + encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() + ); + checkNotNull(encryptedKeyVersion.getEncryptedKeyVersion(), "encryptedKey"); Map params = new HashMap(); params.put(KMSRESTConstants.EEK_OP, KMSRESTConstants.EEK_DECRYPT); Map jsonPayload = new HashMap(); jsonPayload.put(KMSRESTConstants.NAME_FIELD, - encryptedKeyVersion.getKeyName()); + encryptedKeyVersion.getEncryptionKeyName()); jsonPayload.put(KMSRESTConstants.IV_FIELD, Base64.encodeBase64String( - encryptedKeyVersion.getIv())); + encryptedKeyVersion.getEncryptedKeyIv())); jsonPayload.put(KMSRESTConstants.MATERIAL_FIELD, Base64.encodeBase64String( - encryptedKeyVersion.getEncryptedKey().getMaterial())); + encryptedKeyVersion.getEncryptedKeyVersion().getMaterial())); URL url = createURL(KMSRESTConstants.KEY_VERSION_RESOURCE, - encryptedKeyVersion.getKeyVersionName(), + encryptedKeyVersion.getEncryptionKeyVersionName(), KMSRESTConstants.EEK_SUB_RESOURCE, params); HttpURLConnection conn = createConnection(url, HTTP_POST); conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java index 4b578c2377..a0da98b4fc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java @@ -17,51 +17,112 @@ */ package org.apache.hadoop.crypto.key; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - import java.net.URI; import java.security.SecureRandom; +import java.util.Arrays; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.hadoop.conf.Configuration; +import org.junit.BeforeClass; +import org.junit.Test; + + +import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; public class TestKeyProviderCryptoExtension { private static final String CIPHER = "AES"; + private static final String ENCRYPTION_KEY_NAME = "fooKey"; + + private static Configuration conf; + private static KeyProvider kp; + private static KeyProviderCryptoExtension kpExt; + private static KeyProvider.Options options; + private static KeyVersion encryptionKey; + + @BeforeClass + public static void setup() throws Exception { + conf = new Configuration(); + kp = new UserProvider.Factory().createProvider(new URI("user:///"), conf); + kpExt = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp); + options = new KeyProvider.Options(conf); + options.setCipher(CIPHER); + options.setBitLength(128); + encryptionKey = + kp.createKey(ENCRYPTION_KEY_NAME, SecureRandom.getSeed(16), options); + } @Test public void testGenerateEncryptedKey() throws Exception { - Configuration conf = new Configuration(); - KeyProvider kp = - new UserProvider.Factory().createProvider(new URI("user:///"), conf); - KeyProvider.Options options = new KeyProvider.Options(conf); - options.setCipher(CIPHER); - options.setBitLength(128); - KeyProvider.KeyVersion kv = kp.createKey("foo", SecureRandom.getSeed(16), - options); - KeyProviderCryptoExtension kpExt = - KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp); - + // Generate a new EEK and check it KeyProviderCryptoExtension.EncryptedKeyVersion ek1 = - kpExt.generateEncryptedKey(kv.getName()); - Assert.assertEquals(KeyProviderCryptoExtension.EEK, - ek1.getEncryptedKey().getVersionName()); - Assert.assertEquals("foo", ek1.getKeyName()); - Assert.assertNotNull(ek1.getEncryptedKey().getMaterial()); - Assert.assertEquals(kv.getMaterial().length, - ek1.getEncryptedKey().getMaterial().length); - KeyProvider.KeyVersion k1 = kpExt.decryptEncryptedKey(ek1); - Assert.assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName()); - KeyProvider.KeyVersion k1a = kpExt.decryptEncryptedKey(ek1); - Assert.assertArrayEquals(k1.getMaterial(), k1a.getMaterial()); - Assert.assertEquals(kv.getMaterial().length, k1.getMaterial().length); + kpExt.generateEncryptedKey(encryptionKey.getName()); + assertEquals("Version name of EEK should be EEK", + KeyProviderCryptoExtension.EEK, + ek1.getEncryptedKeyVersion().getVersionName()); + assertEquals("Name of EEK should be encryption key name", + ENCRYPTION_KEY_NAME, ek1.getEncryptionKeyName()); + assertNotNull("Expected encrypted key material", + ek1.getEncryptedKeyVersion().getMaterial()); + assertEquals("Length of encryption key material and EEK material should " + + "be the same", encryptionKey.getMaterial().length, + ek1.getEncryptedKeyVersion().getMaterial().length + ); - KeyProviderCryptoExtension.EncryptedKeyVersion ek2 = - kpExt.generateEncryptedKey(kv.getName()); - KeyProvider.KeyVersion k2 = kpExt.decryptEncryptedKey(ek2); - boolean eq = true; - for (int i = 0; eq && i < ek2.getEncryptedKey().getMaterial().length; i++) { - eq = k2.getMaterial()[i] == k1.getMaterial()[i]; + // Decrypt EEK into an EK and check it + KeyVersion k1 = kpExt.decryptEncryptedKey(ek1); + assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName()); + assertEquals(encryptionKey.getMaterial().length, k1.getMaterial().length); + if (Arrays.equals(k1.getMaterial(), encryptionKey.getMaterial())) { + fail("Encrypted key material should not equal encryption key material"); } - Assert.assertFalse(eq); + if (Arrays.equals(ek1.getEncryptedKeyVersion().getMaterial(), + encryptionKey.getMaterial())) { + fail("Encrypted key material should not equal decrypted key material"); + } + // Decrypt it again and it should be the same + KeyVersion k1a = kpExt.decryptEncryptedKey(ek1); + assertArrayEquals(k1.getMaterial(), k1a.getMaterial()); + + // Generate another EEK and make sure it's different from the first + KeyProviderCryptoExtension.EncryptedKeyVersion ek2 = + kpExt.generateEncryptedKey(encryptionKey.getName()); + KeyVersion k2 = kpExt.decryptEncryptedKey(ek2); + if (Arrays.equals(k1.getMaterial(), k2.getMaterial())) { + fail("Generated EEKs should have different material!"); + } + if (Arrays.equals(ek1.getEncryptedKeyIv(), ek2.getEncryptedKeyIv())) { + fail("Generated EEKs should have different IVs!"); + } + } + + @Test + public void testEncryptDecrypt() throws Exception { + // Get an EEK + KeyProviderCryptoExtension.EncryptedKeyVersion eek = + kpExt.generateEncryptedKey(encryptionKey.getName()); + final byte[] encryptedKeyIv = eek.getEncryptedKeyIv(); + final byte[] encryptedKeyMaterial = eek.getEncryptedKeyVersion() + .getMaterial(); + // Decrypt it manually + Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, + new SecretKeySpec(encryptionKey.getMaterial(), "AES"), + new IvParameterSpec(KeyProviderCryptoExtension.EncryptedKeyVersion + .deriveIV(encryptedKeyIv))); + final byte[] manualMaterial = cipher.doFinal(encryptedKeyMaterial); + // Decrypt it with the API + KeyVersion decryptedKey = kpExt.decryptEncryptedKey(eek); + final byte[] apiMaterial = decryptedKey.getMaterial(); + + assertArrayEquals("Wrong key material from decryptEncryptedKey", + manualMaterial, apiMaterial); } } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java index aafb7046fc..24af81be23 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java @@ -64,12 +64,12 @@ public static Map toJSON(EncryptedKeyVersion encryptedKeyVersion) { Map json = new LinkedHashMap(); if (encryptedKeyVersion != null) { json.put(KMSRESTConstants.VERSION_NAME_FIELD, - encryptedKeyVersion.getKeyVersionName()); + encryptedKeyVersion.getEncryptionKeyVersionName()); json.put(KMSRESTConstants.IV_FIELD, Base64.encodeBase64URLSafeString( - encryptedKeyVersion.getIv())); + encryptedKeyVersion.getEncryptedKeyIv())); json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD, - toJSON(encryptedKeyVersion.getEncryptedKey())); + toJSON(encryptedKeyVersion.getEncryptedKeyVersion())); } return json; } diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 26b334df45..021f3cb053 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -485,10 +485,10 @@ public Void call() throws Exception { EncryptedKeyVersion ek1 = kpExt.generateEncryptedKey(kv.getName()); Assert.assertEquals(KeyProviderCryptoExtension.EEK, - ek1.getEncryptedKey().getVersionName()); - Assert.assertNotNull(ek1.getEncryptedKey().getMaterial()); + ek1.getEncryptedKeyVersion().getVersionName()); + Assert.assertNotNull(ek1.getEncryptedKeyVersion().getMaterial()); Assert.assertEquals(kv.getMaterial().length, - ek1.getEncryptedKey().getMaterial().length); + ek1.getEncryptedKeyVersion().getMaterial().length); KeyProvider.KeyVersion k1 = kpExt.decryptEncryptedKey(ek1); Assert.assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName()); KeyProvider.KeyVersion k1a = kpExt.decryptEncryptedKey(ek1); @@ -498,8 +498,8 @@ public Void call() throws Exception { EncryptedKeyVersion ek2 = kpExt.generateEncryptedKey(kv.getName()); KeyProvider.KeyVersion k2 = kpExt.decryptEncryptedKey(ek2); boolean isEq = true; - for (int i = 0; isEq && i < ek2.getEncryptedKey().getMaterial().length; - i++) { + for (int i = 0; isEq && i < ek2.getEncryptedKeyVersion() + .getMaterial().length; i++) { isEq = k2.getMaterial()[i] == k1.getMaterial()[i]; } Assert.assertFalse(isEq);