diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 629ffdac70..2c225cb18e 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -238,7 +238,7 @@
default-war
- package
+ prepare-package
war
@@ -251,6 +251,29 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ prepare-jar
+ prepare-package
+
+ jar
+
+
+ classes
+
+
+
+ prepare-test-jar
+ prepare-package
+
+ test-jar
+
+
+
+
org.codehaus.mojo
findbugs-maven-plugin
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 195eee86ad..f64dcf0e1a 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.fs.Path;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@@ -26,7 +28,10 @@
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.Writer;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -34,6 +39,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.UUID;
public class MiniKMS {
@@ -140,13 +146,15 @@ public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
}
public void start() throws Exception {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
File aclsFile = new File(kmsConfDir, "kms-acls.xml");
if (!aclsFile.exists()) {
- Configuration acls = new Configuration(false);
- Writer writer = new FileWriter(aclsFile);
- acls.writeXml(writer);
- writer.close();
+ InputStream is = cl.getResourceAsStream("mini-kms-acls-default.xml");
+ OutputStream os = new FileOutputStream(aclsFile);
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
}
File coreFile = new File(kmsConfDir, "core-site.xml");
if (!coreFile.exists()) {
@@ -161,19 +169,42 @@ public void start() throws Exception {
kms.set("hadoop.security.key.provider.path",
"jceks://file@" + new Path(kmsConfDir, "kms.keystore").toUri());
kms.set("hadoop.kms.authentication.type", "simple");
+ kms.setBoolean(KMSConfiguration.KEY_AUTHORIZATION_ENABLE, false);
Writer writer = new FileWriter(kmsFile);
kms.writeXml(writer);
writer.close();
}
System.setProperty("log4j.configuration", log4jConfFile);
jetty = createJettyServer(keyStore, keyStorePassword);
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL url = cl.getResource("kms-webapp");
- if (url == null) {
+
+ // we need to do a special handling for MiniKMS to work when in a dir and
+ // when in a JAR in the classpath thanks to Jetty way of handling of webapps
+ // when they are in the a DIR, WAR or JAR.
+ URL webXmlUrl = cl.getResource("kms-webapp/WEB-INF/web.xml");
+ if (webXmlUrl == null) {
throw new RuntimeException(
"Could not find kms-webapp/ dir in test classpath");
}
- WebAppContext context = new WebAppContext(url.getPath(), "/kms");
+ boolean webXmlInJar = webXmlUrl.getPath().contains(".jar!/");
+ String webappPath;
+ if (webXmlInJar) {
+ File webInf = new File("target/" + UUID.randomUUID().toString() +
+ "/kms-webapp/WEB-INF");
+ webInf.mkdirs();
+ new File(webInf, "web.xml").delete();
+ InputStream is = cl.getResourceAsStream("kms-webapp/WEB-INF/web.xml");
+ OutputStream os = new FileOutputStream(new File(webInf, "web.xml"));
+ IOUtils.copy(is, os);
+ is.close();
+ os.close();
+ webappPath = webInf.getParentFile().getAbsolutePath();
+ } else {
+ webappPath = cl.getResource("kms-webapp").getPath();
+ }
+ WebAppContext context = new WebAppContext(webappPath, "/kms");
+ if (webXmlInJar) {
+ context.setClassLoader(cl);
+ }
jetty.addHandler(context);
jetty.start();
kmsURL = new URL(getJettyURL(jetty), "kms");
diff --git a/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml b/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml
new file mode 100644
index 0000000000..24a46b86ec
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml
@@ -0,0 +1,135 @@
+
+
+
+
+
+
+
+
+
+ hadoop.kms.acl.CREATE
+ *
+
+ ACL for create-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+
+
+
+
+ hadoop.kms.acl.DELETE
+ *
+
+ ACL for delete-key operations.
+
+
+
+
+ hadoop.kms.acl.ROLLOVER
+ *
+
+ ACL for rollover-key operations.
+ If the user does is not in the GET ACL, the key material is not returned
+ as part of the response.
+
+
+
+
+ hadoop.kms.acl.GET
+ *
+
+ ACL for get-key-version and get-current-key operations.
+
+
+
+
+ hadoop.kms.acl.GET_KEYS
+ *
+
+ ACL for get-keys operation.
+
+
+
+
+ hadoop.kms.acl.GET_METADATA
+ *
+
+ ACL for get-key-metadata an get-keys-metadata operations.
+
+
+
+
+ hadoop.kms.acl.SET_KEY_MATERIAL
+ *
+
+ Complimentary ACL for CREATE and ROLLOVER operation to allow the client
+ to provide the key material when creating or rolling a key.
+
+
+
+
+ hadoop.kms.acl.GENERATE_EEK
+ *
+
+ ACL for generateEncryptedKey CryptoExtension operations
+
+
+
+
+ hadoop.kms.acl.DECRYPT_EEK
+ *
+
+ ACL for decrypt EncryptedKey CryptoExtension operations
+
+
+
+
+ default.key.acl.MANAGEMENT
+ *
+
+ default ACL for MANAGEMENT operations for all key acls that are not
+ explicitly defined.
+
+
+
+
+ default.key.acl.GENERATE_EEK
+ *
+
+ default ACL for GENERATE_EEK operations for all key acls that are not
+ explicitly defined.
+
+
+
+
+ default.key.acl.DECRYPT_EEK
+ *
+
+ default ACL for DECRYPT_EEK operations for all key acls that are not
+ explicitly defined.
+
+
+
+
+ default.key.acl.READ
+ *
+
+ default ACL for READ operations for all key acls that are not
+ explicitly defined.
+
+
+
+
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1225311ea2..4976938588 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -464,6 +464,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6880. Adding tracing to DataNode data transfer protocol. (iwasakims
via cmccabe)
+ HDFS-7006. Test encryption zones with KMS. (Anthony Young-Garner and tucu)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index ecdd1aeac4..b1707fef00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -185,6 +185,19 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.htrace
htrace-core
+
+ org.apache.hadoop
+ hadoop-kms
+ classes
+ jar
+ test
+
+
+ org.apache.hadoop
+ hadoop-kms
+ test-jar
+ test
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 68fc850652..96f5fcee07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -112,6 +112,11 @@ public class TestEncryptionZones {
protected FileSystemTestWrapper fsWrapper;
protected FileContextTestWrapper fcWrapper;
+ protected String getKeyProviderURI() {
+ return JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir +
+ "/test.jks";
+ }
+
@Before
public void setup() throws Exception {
conf = new HdfsConfiguration();
@@ -119,10 +124,7 @@ public void setup() throws Exception {
// Set up java key store
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
- final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
- JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
- );
+ conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
new file mode 100644
index 0000000000..3a9a89e4a6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.io.File;
+import java.util.UUID;
+
+public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
+
+ private MiniKMS miniKMS;
+
+ @Override
+ protected String getKeyProviderURI() {
+ return KMSClientProvider.SCHEME_NAME + "://" +
+ miniKMS.getKMSUrl().toExternalForm().replace("://", "@");
+ }
+
+ @Before
+ public void setup() throws Exception {
+ File kmsDir = new File("target/test-classes/" +
+ UUID.randomUUID().toString());
+ Assert.assertTrue(kmsDir.mkdirs());
+ MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder();
+ miniKMS = miniKMSBuilder.setKmsConfDir(kmsDir).build();
+ miniKMS.start();
+ super.setup();
+ }
+
+ @After
+ public void teardown() {
+ super.teardown();
+ miniKMS.stop();
+ }
+
+}
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0f662a2049..a121fafc33 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -334,6 +334,20 @@
${project.version}
+
+ org.apache.hadoop
+ hadoop-kms
+ ${project.version}
+ classes
+ jar
+
+
+ org.apache.hadoop
+ hadoop-kms
+ ${project.version}
+ test-jar
+
+
com.google.guava
guava