big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata. (tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604041 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2014-06-19 22:18:03 +00:00
parent 9ff3836a36
commit 38e2322d84
11 changed files with 122 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -152,6 +152,9 @@ Trunk (Unreleased)
HADOOP-10607. Create API to separate credential/password storage from
applications. (Larry McCay via omalley)
HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
(tucu)
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
View File

@ -270,7 +270,7 @@ public KeyVersion createKey(String name, byte[] material,
e);
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
options.getDescription(), new Date(), 1);
options.getDescription(), options.getAttributes(), new Date(), 1);
if (options.getBitLength() != 8 * material.length) {
throw new IOException("Wrong key length. Required " +
options.getBitLength() + ", but got " + (8 * material.length));

48
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
View File

@ -26,8 +26,11 @@
import java.net.URI;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
@ -107,18 +110,22 @@ public static class Metadata {
private final static String CREATED_FIELD = "created";
private final static String DESCRIPTION_FIELD = "description";
private final static String VERSIONS_FIELD = "versions";
private final static String ATTRIBUTES_FIELD = "attributes";
private final String cipher;
private final int bitLength;
private final String description;
private final Date created;
private int versions;
private Map<String, String> attributes;
protected Metadata(String cipher, int bitLength,
String description, Date created, int versions) {
protected Metadata(String cipher, int bitLength, String description,
Map<String, String> attributes, Date created, int versions) {
this.cipher = cipher;
this.bitLength = bitLength;
this.description = description;
this.attributes = (attributes == null || attributes.isEmpty())
? null : attributes;
this.created = created;
this.versions = versions;
}
@ -141,6 +148,11 @@ public String getCipher() {
return cipher;
}
@SuppressWarnings("unchecked")
public Map<String, String> getAttributes() {
return (attributes == null) ? Collections.EMPTY_MAP : attributes;
}
/**
* Get the algorithm from the cipher.
* @return the algorithm name
@ -188,6 +200,13 @@ protected byte[] serialize() throws IOException {
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
if (attributes != null && attributes.size() > 0) {
writer.name(ATTRIBUTES_FIELD).beginObject();
for (Map.Entry<String, String> attribute : attributes.entrySet()) {
writer.name(attribute.getKey()).value(attribute.getValue());
}
writer.endObject();
}
writer.name(VERSIONS_FIELD).value(versions);
writer.endObject();
writer.flush();
@ -208,6 +227,7 @@ protected Metadata(byte[] bytes) throws IOException {
Date created = null;
int versions = 0;
String description = null;
Map<String, String> attributes = null;
JsonReader reader = new JsonReader(new InputStreamReader
(new ByteArrayInputStream(bytes)));
try {
@ -224,6 +244,13 @@ protected Metadata(byte[] bytes) throws IOException {
versions = reader.nextInt();
} else if (DESCRIPTION_FIELD.equals(field)) {
description = reader.nextString();
} else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
reader.beginObject();
attributes = new HashMap<String, String>();
while (reader.hasNext()) {
attributes.put(reader.nextName(), reader.nextString());
}
reader.endObject();
}
}
reader.endObject();
@ -234,6 +261,7 @@ protected Metadata(byte[] bytes) throws IOException {
this.bitLength = bitLength;
this.created = created;
this.description = description;
this.attributes = attributes;
this.versions = versions;
}
}
@ -245,6 +273,7 @@ public static class Options {
private String cipher;
private int bitLength;
private String description;
private Map<String, String> attributes;
public Options(Configuration conf) {
cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
@ -266,6 +295,16 @@ public Options setDescription(String description) {
return this;
}
public Options setAttributes(Map<String, String> attributes) {
if (attributes != null) {
if (attributes.containsKey(null)) {
throw new IllegalArgumentException("attributes cannot have a NULL key");
}
this.attributes = new HashMap<String, String>(attributes);
}
return this;
}
public String getCipher() {
return cipher;
}
@ -277,6 +316,11 @@ public int getBitLength() {
public String getDescription() {
return description;
}
@SuppressWarnings("unchecked")
public Map<String, String> getAttributes() {
return (attributes == null) ? Collections.EMPTY_MAP : attributes;
}
}
/**

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
View File

@ -89,7 +89,7 @@ public synchronized KeyVersion createKey(String name, byte[] material,
options.getBitLength() + ", but got " + (8 * material.length));
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
options.getDescription(), new Date(), 1);
options.getDescription(), options.getAttributes(), new Date(), 1);
cache.put(name, meta);
String versionName = buildVersionName(name, 0);
credentials.addSecretKey(nameT, meta.serialize());

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
View File

@ -83,6 +83,7 @@ private static KeyVersion parseJSONKeyVersion(Map valueMap) {
return keyVersion;
}
@SuppressWarnings("unchecked")
private static Metadata parseJSONMetadata(Map valueMap) {
Metadata metadata = null;
if (!valueMap.isEmpty()) {
@ -90,6 +91,7 @@ private static Metadata parseJSONMetadata(Map valueMap) {
(String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
(Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
(String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
(Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
(Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
}
@ -351,8 +353,8 @@ public List<String> getKeys() throws IOException {
public static class KMSMetadata extends Metadata {
public KMSMetadata(String cipher, int bitLength, String description,
Date created, int versions) {
super(cipher, bitLength, description, created, versions);
Map<String, String> attributes, Date created, int versions) {
super(cipher, bitLength, description, attributes, created, versions);
}
}
@ -416,6 +418,9 @@ private KeyVersion createKeyInternal(String name, byte[] material,
jsonKey.put(KMSRESTConstants.DESCRIPTION_FIELD,
options.getDescription());
}
if (options.getAttributes() != null && !options.getAttributes().isEmpty()) {
jsonKey.put(KMSRESTConstants.ATTRIBUTES_FIELD, options.getAttributes());
}
URL url = createURL(KMSRESTConstants.KEYS_RESOURCE, null, null, null);
HttpURLConnection conn = createConnection(url, HTTP_POST);
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
View File

@ -42,6 +42,7 @@ public class KMSRESTConstants {
public static final String CIPHER_FIELD = "cipher";
public static final String LENGTH_FIELD = "length";
public static final String DESCRIPTION_FIELD = "description";
public static final String ATTRIBUTES_FIELD = "attributes";
public static final String CREATED_FIELD = "created";
public static final String VERSIONS_FIELD = "versions";
public static final String MATERIAL_FIELD = "material";

17
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
View File

@ -30,7 +30,9 @@
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@ -73,7 +75,7 @@ public void testMetadata() throws Exception {
DateFormat format = new SimpleDateFormat("y/m/d");
Date date = format.parse("2013/12/25");
KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null,
date, 123);
null, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertNull(meta.getDescription());
@ -83,6 +85,7 @@ public void testMetadata() throws Exception {
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertNull(second.getDescription());
assertTrue(second.getAttributes().isEmpty());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
int newVersion = second.addVersion();
@ -93,17 +96,21 @@ public void testMetadata() throws Exception {
//Metadata with description
format = new SimpleDateFormat("y/m/d");
date = format.parse("2013/12/25");
Map<String, String> attributes = new HashMap<String, String>();
attributes.put("a", "A");
meta = new KeyProvider.Metadata("myCipher", 100,
"description", date, 123);
"description", attributes, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertEquals("description", meta.getDescription());
assertEquals(attributes, meta.getAttributes());
assertEquals(date, meta.getCreated());
assertEquals(123, meta.getVersions());
second = new KeyProvider.Metadata(meta.serialize());
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertEquals(meta.getDescription(), second.getDescription());
assertEquals(meta.getAttributes(), second.getAttributes());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
newVersion = second.addVersion();
@ -117,15 +124,19 @@ public void testOptions() throws Exception {
Configuration conf = new Configuration();
conf.set(KeyProvider.DEFAULT_CIPHER_NAME, "myCipher");
conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 512);
Map<String, String> attributes = new HashMap<String, String>();
attributes.put("a", "A");
KeyProvider.Options options = KeyProvider.options(conf);
assertEquals("myCipher", options.getCipher());
assertEquals(512, options.getBitLength());
options.setCipher("yourCipher");
options.setDescription("description");
options.setAttributes(attributes);
options.setBitLength(128);
assertEquals("yourCipher", options.getCipher());
assertEquals(128, options.getBitLength());
assertEquals("description", options.getDescription());
assertEquals(attributes, options.getAttributes());
options = KeyProvider.options(new Configuration());
assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher());
assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength());
@ -167,7 +178,7 @@ public List<KeyVersion> getKeyVersions(String name)
@Override
public Metadata getMetadata(String name) throws IOException {
return new Metadata(CIPHER, 128, "description", new Date(), 0);
return new Metadata(CIPHER, 128, "description", null, new Date(), 0);
}
@Override

5
hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
View File

@ -103,6 +103,7 @@ private static URI getKeyURI(String name) throws URISyntaxException {
@Path(KMSRESTConstants.KEYS_RESOURCE)
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings("unchecked")
public Response createKey(@Context SecurityContext securityContext,
Map jsonKey) throws Exception {
KMSWebApp.getAdminCallsMeter().mark();
@ -116,7 +117,8 @@ public Response createKey(@Context SecurityContext securityContext,
? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
String description = (String)
jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
Map<String, String> attributes = (Map<String, String>)
jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
if (material != null) {
assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user,
CREATE_KEY + " with user provided material", name);
@ -130,6 +132,7 @@ public Response createKey(@Context SecurityContext securityContext,
options.setBitLength(length);
}
options.setDescription(description);
options.setAttributes(attributes);
KeyProvider.KeyVersion keyVersion = (material != null)
? provider.createKey(name, Base64.decodeBase64(material), options)

1
hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
View File

@ -61,6 +61,7 @@ public static Map toJSON(String keyName, KeyProvider.Metadata meta) {
json.put(KMSRESTConstants.CIPHER_FIELD, meta.getCipher());
json.put(KMSRESTConstants.LENGTH_FIELD, meta.getBitLength());
json.put(KMSRESTConstants.DESCRIPTION_FIELD, meta.getDescription());
json.put(KMSRESTConstants.ATTRIBUTES_FIELD, meta.getAttributes());
json.put(KMSRESTConstants.CREATED_FIELD,
meta.getCreated().getTime());
json.put(KMSRESTConstants.VERSIONS_FIELD,

43
hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
View File

@ -490,6 +490,49 @@ public Void call() throws Exception {
// getKeysMetadata() empty
Assert.assertEquals(0, kp.getKeysMetadata().length);
// createKey() no description, no tags
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
kp.createKey("k2", options);
KeyProvider.Metadata meta = kp.getMetadata("k2");
Assert.assertNull(meta.getDescription());
Assert.assertTrue(meta.getAttributes().isEmpty());
// createKey() description, no tags
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
options.setDescription("d");
kp.createKey("k3", options);
meta = kp.getMetadata("k3");
Assert.assertEquals("d", meta.getDescription());
Assert.assertTrue(meta.getAttributes().isEmpty());
Map<String, String> attributes = new HashMap<String, String>();
attributes.put("a", "A");
// createKey() no description, tags
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
options.setAttributes(attributes);
kp.createKey("k4", options);
meta = kp.getMetadata("k4");
Assert.assertNull(meta.getDescription());
Assert.assertEquals(attributes, meta.getAttributes());
// createKey() description, tags
options = new KeyProvider.Options(conf);
options.setCipher("AES/CTR/NoPadding");
options.setBitLength(128);
options.setDescription("d");
options.setAttributes(attributes);
kp.createKey("k5", options);
meta = kp.getMetadata("k5");
Assert.assertEquals("d", meta.getDescription());
Assert.assertEquals(attributes, meta.getAttributes());
return null;
}
});

2
hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
View File

@ -102,7 +102,7 @@ public void testDeleteKey() throws Exception {
Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
Mockito.when(mockProv.getKeyVersion(Mockito.eq("k1@0"))).thenReturn(mockKey);
Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(
new KMSClientProvider.KMSMetadata("c", 0, "l", new Date(), 1));
new KMSClientProvider.KMSMetadata("c", 0, "l", null, new Date(), 1));
KeyProvider cache = new KMSCacheKeyProvider(mockProv, 100);
Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));