HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
This commit is contained in:
parent
ffdb7eb3b2
commit
3e85f5b605
@ -238,7 +238,7 @@
|
|||||||
<executions>
|
<executions>
|
||||||
<execution>
|
<execution>
|
||||||
<id>default-war</id>
|
<id>default-war</id>
|
||||||
<phase>package</phase>
|
<phase>prepare-package</phase>
|
||||||
<goals>
|
<goals>
|
||||||
<goal>war</goal>
|
<goal>war</goal>
|
||||||
</goals>
|
</goals>
|
||||||
@ -251,6 +251,29 @@
|
|||||||
</execution>
|
</execution>
|
||||||
</executions>
|
</executions>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>prepare-jar</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>jar</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<classifier>classes</classifier>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>prepare-test-jar</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>test-jar</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.codehaus.mojo</groupId>
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
<artifactId>findbugs-maven-plugin</artifactId>
|
<artifactId>findbugs-maven-plugin</artifactId>
|
||||||
|
@ -18,7 +18,9 @@
|
|||||||
package org.apache.hadoop.crypto.key.kms.server;
|
package org.apache.hadoop.crypto.key.kms.server;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.mortbay.jetty.Connector;
|
import org.mortbay.jetty.Connector;
|
||||||
import org.mortbay.jetty.Server;
|
import org.mortbay.jetty.Server;
|
||||||
@ -26,7 +28,10 @@
|
|||||||
import org.mortbay.jetty.webapp.WebAppContext;
|
import org.mortbay.jetty.webapp.WebAppContext;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
import java.io.Writer;
|
import java.io.Writer;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
@ -34,6 +39,7 @@
|
|||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
public class MiniKMS {
|
public class MiniKMS {
|
||||||
|
|
||||||
@ -140,13 +146,15 @@ public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void start() throws Exception {
|
public void start() throws Exception {
|
||||||
|
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
||||||
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
|
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
|
||||||
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
|
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
|
||||||
if (!aclsFile.exists()) {
|
if (!aclsFile.exists()) {
|
||||||
Configuration acls = new Configuration(false);
|
InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
|
||||||
Writer writer = new FileWriter(aclsFile);
|
OutputStream os = new FileOutputStream(aclsFile);
|
||||||
acls.writeXml(writer);
|
IOUtils.copy(is, os);
|
||||||
writer.close();
|
is.close();
|
||||||
|
os.close();
|
||||||
}
|
}
|
||||||
File coreFile = new File(kmsConfDir, "core-site.xml");
|
File coreFile = new File(kmsConfDir, "core-site.xml");
|
||||||
if (!coreFile.exists()) {
|
if (!coreFile.exists()) {
|
||||||
@ -161,19 +169,42 @@ public void start() throws Exception {
|
|||||||
kms.set("hadoop.security.key.provider.path",
|
kms.set("hadoop.security.key.provider.path",
|
||||||
"jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
|
"jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
|
||||||
kms.set("hadoop.kms.authentication.type", "simple");
|
kms.set("hadoop.kms.authentication.type", "simple");
|
||||||
|
kms.setBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE, false);
|
||||||
Writer writer = new FileWriter(kmsFile);
|
Writer writer = new FileWriter(kmsFile);
|
||||||
kms.writeXml(writer);
|
kms.writeXml(writer);
|
||||||
writer.close();
|
writer.close();
|
||||||
}
|
}
|
||||||
System.setProperty("log4j.configuration", log4jConfFile);
|
System.setProperty("log4j.configuration", log4jConfFile);
|
||||||
jetty = createJettyServer(keyStore, keyStorePassword);
|
jetty = createJettyServer(keyStore, keyStorePassword);
|
||||||
ClassLoader cl = Thread.currentThread().getContextClassLoader();
|
|
||||||
URL url = cl.getResource("kms-webapp");
|
// we need to do a special handling for MiniKMS to work when in a dir and
|
||||||
if (url == null) {
|
// when in a JAR in the classpath thanks to Jetty way of handling of webapps
|
||||||
|
// when they are in the a DIR, WAR or JAR.
|
||||||
|
URL webXmlUrl = cl.getResource("kms-webapp/WEB-INF/web.xml");
|
||||||
|
if (webXmlUrl == null) {
|
||||||
throw new RuntimeException(
|
throw new RuntimeException(
|
||||||
"Could not find kms-webapp/ dir in test classpath");
|
"Could not find kms-webapp/ dir in test classpath");
|
||||||
}
|
}
|
||||||
WebAppContext context = new WebAppContext(url.getPath(), "/kms");
|
boolean webXmlInJar = webXmlUrl.getPath().contains(".jar!/");
|
||||||
|
String webappPath;
|
||||||
|
if (webXmlInJar) {
|
||||||
|
File webInf = new File("target/" + UUID.randomUUID().toString() +
|
||||||
|
"/kms-webapp/WEB-INF");
|
||||||
|
webInf.mkdirs();
|
||||||
|
new File(webInf, "web.xml").delete();
|
||||||
|
InputStream is = cl.getResourceAsStream("kms-webapp/WEB-INF/web.xml");
|
||||||
|
OutputStream os = new FileOutputStream(new File(webInf, "web.xml"));
|
||||||
|
IOUtils.copy(is, os);
|
||||||
|
is.close();
|
||||||
|
os.close();
|
||||||
|
webappPath = webInf.getParentFile().getAbsolutePath();
|
||||||
|
} else {
|
||||||
|
webappPath = cl.getResource("kms-webapp").getPath();
|
||||||
|
}
|
||||||
|
WebAppContext context = new WebAppContext(webappPath, "/kms");
|
||||||
|
if (webXmlInJar) {
|
||||||
|
context.setClassLoader(cl);
|
||||||
|
}
|
||||||
jetty.addHandler(context);
|
jetty.addHandler(context);
|
||||||
jetty.start();
|
jetty.start();
|
||||||
kmsURL = new URL(getJettyURL(jetty), "kms");
|
kmsURL = new URL(getJettyURL(jetty), "kms");
|
||||||
|
@ -0,0 +1,135 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<!-- This file is hot-reloaded when it changes -->
|
||||||
|
|
||||||
|
<!-- KMS ACLs -->
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.CREATE</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for create-key operations.
|
||||||
|
If the user does is not in the GET ACL, the key material is not returned
|
||||||
|
as part of the response.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.DELETE</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for delete-key operations.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.ROLLOVER</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for rollover-key operations.
|
||||||
|
If the user does is not in the GET ACL, the key material is not returned
|
||||||
|
as part of the response.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.GET</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for get-key-version and get-current-key operations.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.GET_KEYS</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for get-keys operation.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.GET_METADATA</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for get-key-metadata an get-keys-metadata operations.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
Complimentary ACL for CREATE and ROLLOVER operation to allow the client
|
||||||
|
to provide the key material when creating or rolling a key.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.GENERATE_EEK</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for generateEncryptedKey CryptoExtension operations
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.kms.acl.DECRYPT_EEK</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
ACL for decrypt EncryptedKey CryptoExtension operations
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>default.key.acl.MANAGEMENT</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
default ACL for MANAGEMENT operations for all key acls that are not
|
||||||
|
explicitly defined.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>default.key.acl.GENERATE_EEK</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
default ACL for GENERATE_EEK operations for all key acls that are not
|
||||||
|
explicitly defined.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>default.key.acl.DECRYPT_EEK</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
default ACL for DECRYPT_EEK operations for all key acls that are not
|
||||||
|
explicitly defined.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>default.key.acl.READ</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>
|
||||||
|
default ACL for READ operations for all key acls that are not
|
||||||
|
explicitly defined.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
|
||||||
|
</configuration>
|
@ -464,6 +464,8 @@ Release 2.6.0 - UNRELEASED
|
|||||||
HDFS-6880. Adding tracing to DataNode data transfer protocol. (iwasakims
|
HDFS-6880. Adding tracing to DataNode data transfer protocol. (iwasakims
|
||||||
via cmccabe)
|
via cmccabe)
|
||||||
|
|
||||||
|
HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
||||||
|
@ -185,6 +185,19 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||||||
<groupId>org.htrace</groupId>
|
<groupId>org.htrace</groupId>
|
||||||
<artifactId>htrace-core</artifactId>
|
<artifactId>htrace-core</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-kms</artifactId>
|
||||||
|
<classifier>classes</classifier>
|
||||||
|
<type>jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-kms</artifactId>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
@ -112,6 +112,11 @@ public class TestEncryptionZones {
|
|||||||
protected FileSystemTestWrapper fsWrapper;
|
protected FileSystemTestWrapper fsWrapper;
|
||||||
protected FileContextTestWrapper fcWrapper;
|
protected FileContextTestWrapper fcWrapper;
|
||||||
|
|
||||||
|
protected String getKeyProviderURI() {
|
||||||
|
return JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir +
|
||||||
|
"/test.jks";
|
||||||
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setup() throws Exception {
|
public void setup() throws Exception {
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
@ -119,10 +124,7 @@ public void setup() throws Exception {
|
|||||||
// Set up java key store
|
// Set up java key store
|
||||||
String testRoot = fsHelper.getTestRootDir();
|
String testRoot = fsHelper.getTestRootDir();
|
||||||
testRootDir = new File(testRoot).getAbsoluteFile();
|
testRootDir = new File(testRoot).getAbsoluteFile();
|
||||||
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
|
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, getKeyProviderURI());
|
||||||
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
|
|
||||||
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
|
|
||||||
);
|
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
|
||||||
// Lower the batch size for testing
|
// Lower the batch size for testing
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
|
||||||
|
@ -0,0 +1,56 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
|
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
|
||||||
|
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
|
||||||
|
|
||||||
|
private MiniKMS miniKMS;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getKeyProviderURI() {
|
||||||
|
return KMSClientProvider.SCHEME_NAME + "://" +
|
||||||
|
miniKMS.getKMSUrl().toExternalForm().replace("://", "@");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws Exception {
|
||||||
|
File kmsDir = new File("target/test-classes/" +
|
||||||
|
UUID.randomUUID().toString());
|
||||||
|
Assert.assertTrue(kmsDir.mkdirs());
|
||||||
|
MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder();
|
||||||
|
miniKMS = miniKMSBuilder.setKmsConfDir(kmsDir).build();
|
||||||
|
miniKMS.start();
|
||||||
|
super.setup();
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void teardown() {
|
||||||
|
super.teardown();
|
||||||
|
miniKMS.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -334,6 +334,20 @@
|
|||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-kms</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<classifier>classes</classifier>
|
||||||
|
<type>jar</type>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-kms</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
<artifactId>guava</artifactId>
|
<artifactId>guava</artifactId>
|
||||||
|
Loading…
Reference in New Issue
Block a user