big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)

Browse Source
This commit is contained in:
Alejandro Abdelnur 2014-09-16 12:39:17 -07:00
parent ffdb7eb3b2
commit 3e85f5b605
8 changed files with 289 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-common-project/hadoop-kms/pom.xml
View File

@ -238,7 +238,7 @@
<executions>
<execution>
<id>default-war</id>
<phase>package</phase>
<phase>prepare-package</phase>
<goals>
<goal>war</goal>
</goals>
@ -251,6 +251,29 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>prepare-jar</id>
<phase>prepare-package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<classifier>classes</classifier>
</configuration>
</execution>
<execution>
<id>prepare-test-jar</id>
<phase>prepare-package</phase>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>

47
hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
View File

@ -18,7 +18,9 @@
package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.fs.Path;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@ -26,7 +28,10 @@
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
import java.net.InetAddress;
import java.net.MalformedURLException;
@ -34,6 +39,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.UUID;
public class MiniKMS {
@ -140,13 +146,15 @@ public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
}
public void start() throws Exception {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
if (!aclsFile.exists()) {
Configuration acls = new Configuration(false);
Writer writer = new FileWriter(aclsFile);
acls.writeXml(writer);
writer.close();
InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
OutputStream os = new FileOutputStream(aclsFile);
IOUtils.copy(is, os);
is.close();
os.close();
}
File coreFile = new File(kmsConfDir, "core-site.xml");
if (!coreFile.exists()) {
@ -161,19 +169,42 @@ public void start() throws Exception {
kms.set("hadoop.security.key.provider.path",
"jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
kms.set("hadoop.kms.authentication.type", "simple");
kms.setBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE, false);
Writer writer = new FileWriter(kmsFile);
kms.writeXml(writer);
writer.close();
}
System.setProperty("log4j.configuration", log4jConfFile);
jetty = createJettyServer(keyStore, keyStorePassword);
ClassLoader cl = Thread.currentThread().getContextClassLoader();
URL url = cl.getResource("kms-webapp");
if (url == null) {
// we need to do a special handling for MiniKMS to work when in a dir and
// when in a JAR in the classpath thanks to Jetty way of handling of webapps
// when they are in the a DIR, WAR or JAR.
URL webXmlUrl = cl.getResource("kms-webapp/WEB-INF/web.xml");
if (webXmlUrl == null) {
throw new RuntimeException(
"Could not find kms-webapp/ dir in test classpath");
}
WebAppContext context = new WebAppContext(url.getPath(), "/kms");
boolean webXmlInJar = webXmlUrl.getPath().contains(".jar!/");
String webappPath;
if (webXmlInJar) {
File webInf = new File("target/" + UUID.randomUUID().toString() +
"/kms-webapp/WEB-INF");
webInf.mkdirs();
new File(webInf, "web.xml").delete();
InputStream is = cl.getResourceAsStream("kms-webapp/WEB-INF/web.xml");
OutputStream os = new FileOutputStream(new File(webInf, "web.xml"));
IOUtils.copy(is, os);
is.close();
os.close();
webappPath = webInf.getParentFile().getAbsolutePath();
} else {
webappPath = cl.getResource("kms-webapp").getPath();
}
WebAppContext context = new WebAppContext(webappPath, "/kms");
if (webXmlInJar) {
context.setClassLoader(cl);
}
jetty.addHandler(context);
jetty.start();
kmsURL = new URL(getJettyURL(jetty), "kms");

135
hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml Normal file
View File

@ -0,0 +1,135 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<!-- This file is hot-reloaded when it changes -->
<!-- KMS ACLs -->
<property>
<name>hadoop.kms.acl.CREATE</name>
<value>*</value>
<description>
ACL for create-key operations.
If the user does is not in the GET ACL, the key material is not returned
as part of the response.
</description>
</property>
<property>
<name>hadoop.kms.acl.DELETE</name>
<value>*</value>
<description>
ACL for delete-key operations.
</description>
</property>
<property>
<name>hadoop.kms.acl.ROLLOVER</name>
<value>*</value>
<description>
ACL for rollover-key operations.
If the user does is not in the GET ACL, the key material is not returned
as part of the response.
</description>
</property>
<property>
<name>hadoop.kms.acl.GET</name>
<value>*</value>
<description>
ACL for get-key-version and get-current-key operations.
</description>
</property>
<property>
<name>hadoop.kms.acl.GET_KEYS</name>
<value>*</value>
<description>
ACL for get-keys operation.
</description>
</property>
<property>
<name>hadoop.kms.acl.GET_METADATA</name>
<value>*</value>
<description>
ACL for get-key-metadata an get-keys-metadata operations.
</description>
</property>
<property>
<name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
<value>*</value>
<description>
Complimentary ACL for CREATE and ROLLOVER operation to allow the client
to provide the key material when creating or rolling a key.
</description>
</property>
<property>
<name>hadoop.kms.acl.GENERATE_EEK</name>
<value>*</value>
<description>
ACL for generateEncryptedKey CryptoExtension operations
</description>
</property>
<property>
<name>hadoop.kms.acl.DECRYPT_EEK</name>
<value>*</value>
<description>
ACL for decrypt EncryptedKey CryptoExtension operations
</description>
</property>
<property>
<name>default.key.acl.MANAGEMENT</name>
<value>*</value>
<description>
default ACL for MANAGEMENT operations for all key acls that are not
explicitly defined.
</description>
</property>
<property>
<name>default.key.acl.GENERATE_EEK</name>
<value>*</value>
<description>
default ACL for GENERATE_EEK operations for all key acls that are not
explicitly defined.
</description>
</property>
<property>
<name>default.key.acl.DECRYPT_EEK</name>
<value>*</value>
<description>
default ACL for DECRYPT_EEK operations for all key acls that are not
explicitly defined.
</description>
</property>
<property>
<name>default.key.acl.READ</name>
<value>*</value>
<description>
default ACL for READ operations for all key acls that are not
explicitly defined.
</description>
</property>
</configuration>

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -464,6 +464,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6880. Adding tracing to DataNode data transfer protocol. (iwasakims
via cmccabe)
HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)

13
hadoop-hdfs-project/hadoop-hdfs/pom.xml
View File

@ -185,6 +185,19 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.htrace</groupId>
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<classifier>classes</classifier>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
View File

@ -112,6 +112,11 @@ public class TestEncryptionZones {
protected FileSystemTestWrapper fsWrapper;
protected FileContextTestWrapper fcWrapper;
protected String getKeyProviderURI() {
return JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir +
"/test.jks";
}
@Before
public void setup() throws Exception {
conf = new HdfsConfiguration();
@ -119,10 +124,7 @@ public void setup() throws Exception {
// Set up java key store
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
);
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,

56
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java Normal file
View File

@ -0,0 +1,56 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import java.io.File;
import java.util.UUID;
public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
private MiniKMS miniKMS;
@Override
protected String getKeyProviderURI() {
return KMSClientProvider.SCHEME_NAME + "://" +
miniKMS.getKMSUrl().toExternalForm().replace("://", "@");
}
@Before
public void setup() throws Exception {
File kmsDir = new File("target/test-classes/" +
UUID.randomUUID().toString());
Assert.assertTrue(kmsDir.mkdirs());
MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder();
miniKMS = miniKMSBuilder.setKmsConfDir(kmsDir).build();
miniKMS.start();
super.setup();
}
@After
public void teardown() {
super.teardown();
miniKMS.stop();
}
}

14
hadoop-project/pom.xml
View File

@ -334,6 +334,20 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>${project.version}</version>
<classifier>classes</classifier>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>