From adf0b67a7104bd457b20c95ff78dd48753dcd699 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 19 Sep 2014 14:54:37 -0700 Subject: [PATCH] HADOOP-10970. Cleanup KMS configuration keys. (wang) --- .../hadoop-common/CHANGES.txt | 2 + .../hadoop-kms/src/main/conf/kms-acls.xml | 12 ++--- .../hadoop-kms/src/main/conf/kms-site.xml | 47 ++++++++++++++++++- .../crypto/key/kms/server/KMSAudit.java | 14 ++++-- .../key/kms/server/KMSConfiguration.java | 6 +-- .../crypto/key/kms/server/KMSWebApp.java | 4 +- 6 files changed, 70 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7fc3b5b098..0603304a7c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -535,6 +535,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11106. Document considerations of HAR and Encryption. (clamb via wang) + HADOOP-10970. Cleanup KMS configuration keys. (wang) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml index 24a46b86ec..1d5b649c83 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml @@ -23,7 +23,7 @@ * ACL for create-key operations. - If the user does is not in the GET ACL, the key material is not returned + If the user is not in the GET ACL, the key material is not returned as part of the response. @@ -58,7 +58,7 @@ hadoop.kms.acl.GET_KEYS * - ACL for get-keys operation. + ACL for get-keys operations. @@ -66,7 +66,7 @@ hadoop.kms.acl.GET_METADATA * - ACL for get-key-metadata an get-keys-metadata operations. + ACL for get-key-metadata and get-keys-metadata operations. @@ -74,7 +74,7 @@ hadoop.kms.acl.SET_KEY_MATERIAL * - Complimentary ACL for CREATE and ROLLOVER operation to allow the client + Complementary ACL for CREATE and ROLLOVER operations to allow the client to provide the key material when creating or rolling a key. @@ -83,7 +83,7 @@ hadoop.kms.acl.GENERATE_EEK * - ACL for generateEncryptedKey CryptoExtension operations + ACL for generateEncryptedKey CryptoExtension operations. @@ -91,7 +91,7 @@ hadoop.kms.acl.DECRYPT_EEK * - ACL for decrypt EncryptedKey CryptoExtension operations + ACL for decryptEncryptedKey CryptoExtension operations. diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 4f4694c3b8..a810ca44d2 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -15,10 +15,12 @@ + hadoop.kms.key.provider.uri jceks://file@/${user.home}/kms.keystore + URI of the backing KeyProvider for the KMS. @@ -26,14 +28,52 @@ hadoop.security.keystore.JavaKeyStoreProvider.password none + If using the JavaKeyStoreProvider, the password for the keystore file. + + + hadoop.kms.cache.enable + true + + Whether the KMS will act as a cache for the backing KeyProvider. + When the cache is enabled, operations like getKeyVersion, getMetadata, + and getCurrentKey will sometimes return cached data without consulting + the backing KeyProvider. Cached values are flushed when keys are deleted + or modified. + + + hadoop.kms.cache.timeout.ms + 600000 + + Expiry time for the KMS key version and key metadata cache, in + milliseconds. This affects getKeyVersion and getMetadata. + + + + + hadoop.kms.current.key.cache.timeout.ms + 30000 + + Expiry time for the KMS current key cache, in milliseconds. This + affects getCurrentKey operations. + + + + + + + hadoop.kms.audit.aggregation.window.ms 10000 + Duplicate audit log events within the aggregation window (specified in + ms) are quashed to reduce log traffic. A single message for aggregated + events is printed at the end of the window, along with a count of the + number of aggregated events. @@ -43,7 +83,8 @@ hadoop.kms.authentication.type simple - simple or kerberos + Authentication type for the KMS. Can be either "simple" + or "kerberos". @@ -51,6 +92,7 @@ hadoop.kms.authentication.kerberos.keytab ${user.home}/kms.keytab + Path to the keytab with credentials for the configured Kerberos principal. @@ -58,6 +100,8 @@ hadoop.kms.authentication.kerberos.principal HTTP/localhost + The Kerberos principal to use for the HTTP endpoint. + The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification. @@ -65,6 +109,7 @@ hadoop.kms.authentication.kerberos.name.rules DEFAULT + Rules used to resolve Kerberos principal names. diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java index dc55a8459c..7ff76e54ff 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java @@ -103,9 +103,17 @@ public static enum OpStatus { private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME); - KMSAudit(long delay) { + /** + * Create a new KMSAudit. + * + * @param windowMs Duplicate events within the aggregation window are quashed + * to reduce log traffic. A single message for aggregated + * events is printed at the end of the window, along with a + * count of the number of aggregated events. + */ + KMSAudit(long windowMs) { cache = CacheBuilder.newBuilder() - .expireAfterWrite(delay, TimeUnit.MILLISECONDS) + .expireAfterWrite(windowMs, TimeUnit.MILLISECONDS) .removalListener( new RemovalListener() { @Override @@ -126,7 +134,7 @@ public void onRemoval( public void run() { cache.cleanUp(); } - }, delay / 10, delay / 10, TimeUnit.MILLISECONDS); + }, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS); } private void logEvent(AuditEvent event) { diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java index c9b04915fd..bd61ca7edf 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java @@ -54,8 +54,8 @@ public class KMSConfiguration { public static final String CURR_KEY_CACHE_TIMEOUT_KEY = CONFIG_PREFIX + "current.key.cache.timeout.ms"; // Delay for Audit logs that need aggregation - public static final String KMS_AUDIT_AGGREGATION_DELAY = CONFIG_PREFIX + - "aggregation.delay.ms"; + public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX + + "audit.aggregation.window.ms"; public static final boolean KEY_CACHE_ENABLE_DEFAULT = true; // 10 mins @@ -63,7 +63,7 @@ public class KMSConfiguration { // 30 secs public static final long CURR_KEY_CACHE_TIMEOUT_DEFAULT = 30 * 1000; // 10 secs - public static final long KMS_AUDIT_AGGREGATION_DELAY_DEFAULT = 10000; + public static final long KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT = 10000; // Property to Enable/Disable per Key authorization public static final String KEY_AUTHORIZATION_ENABLE = CONFIG_PREFIX + diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java index c9eeb1dec8..325f8db27a 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java @@ -148,8 +148,8 @@ public void contextInitialized(ServletContextEvent sce) { kmsAudit = new KMSAudit(kmsConf.getLong( - KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY, - KMSConfiguration.KMS_AUDIT_AGGREGATION_DELAY_DEFAULT)); + KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW, + KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT)); // this is required for the the JMXJsonServlet to work properly. // the JMXJsonServlet is behind the authentication filter,