diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ba960b4fba..eb0c76315c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -135,6 +135,9 @@ Trunk (Unreleased) HADOOP-10429. KeyStores should have methods to generate the materials themselves, KeyShell should use them. (tucu) + HADOOP-10428. JavaKeyStoreProvider should accept keystore password via + configuration falling back to ENV VAR. (tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index a96a9c6bab..b4e623a0d8 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -483,6 +483,7 @@ src/test/resources/test.har/_index src/test/resources/test.har/_masterindex src/test/resources/test.har/part-0 + src/test/resources/javakeystoreprovider.password diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java index eeeaca1bc6..24be5d7068 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java @@ -18,6 +18,7 @@ package org.apache.hadoop.crypto.key; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -27,10 +28,12 @@ import org.apache.hadoop.fs.permission.FsPermission; import javax.crypto.spec.SecretKeySpec; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; import java.net.URI; +import java.net.URL; import java.security.Key; import java.security.KeyStore; import java.security.KeyStoreException; @@ -52,10 +55,21 @@ * any Hadoop FileSystem using the following name mangling: * jks://hdfs@nn1.example.com/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks * jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks - * - * The password for the keystore is taken from the HADOOP_KEYSTORE_PASSWORD - * environment variable with a default of 'none'. - * + *

+ * If the HADOOP_KEYSTORE_PASSWORD environment variable is set, + * its value is used as the password for the keystore. + *

+ * If the HADOOP_KEYSTORE_PASSWORD environment variable is not set, + * the password for the keystore is read from file specified in the + * {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file + * is looked up in Hadoop's configuration directory via the classpath. + *

+ * NOTE: Make sure the password in the password file does not have an + * ENTER at the end, else it won't be valid for the Java KeyStore. + *

+ * If the environment variable, nor the property are not set, the password used + * is 'none'. + *

* It is expected for encrypted InputFormats and OutputFormats to copy the keys * from the original provider into the job's Credentials object, which is * accessed via the UserProvider. Therefore, this provider won't be used by @@ -65,16 +79,20 @@ public class JavaKeyStoreProvider extends KeyProvider { private static final String KEY_METADATA = "KeyMetadata"; public static final String SCHEME_NAME = "jceks"; - public static final String KEYSTORE_PASSWORD_NAME = + + public static final String KEYSTORE_PASSWORD_FILE_KEY = + "hadoop.security.keystore.java-keystore-provider.password-file"; + + public static final String KEYSTORE_PASSWORD_ENV_VAR = "HADOOP_KEYSTORE_PASSWORD"; - public static final String KEYSTORE_PASSWORD_DEFAULT = "none"; + public static final char[] KEYSTORE_PASSWORD_DEFAULT = "none".toCharArray(); private final URI uri; private final Path path; private final FileSystem fs; private final FsPermission permissions; private final KeyStore keyStore; - private final char[] password; + private char[] password; private boolean changed = false; private Lock readLock; private Lock writeLock; @@ -85,12 +103,29 @@ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException { this.uri = uri; path = unnestUri(uri); fs = path.getFileSystem(conf); - // Get the password from the user's environment - String pw = System.getenv(KEYSTORE_PASSWORD_NAME); - if (pw == null) { - pw = KEYSTORE_PASSWORD_DEFAULT; + // Get the password file from the conf, if not present from the user's + // environment var + if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) { + password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray(); + } + if (password == null) { + String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY); + if (pwFile != null) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL pwdFile = cl.getResource(pwFile); + if (pwdFile != null) { + InputStream is = pwdFile.openStream(); + try { + password = IOUtils.toCharArray(is); + } finally { + is.close(); + } + } + } + } + if (password == null) { + password = KEYSTORE_PASSWORD_DEFAULT; } - password = pw.toCharArray(); try { keyStore = KeyStore.getInstance(SCHEME_NAME); if (fs.exists(path)) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java index 4fd5b9bddf..5bede60188 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java @@ -30,6 +30,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; @@ -237,4 +238,48 @@ public void checkPermissionRetention(Configuration conf, String ourUrl, Path pat FileStatus s = fs.getFileStatus(path); assertTrue("Permissions should have been retained from the preexisting keystore.", s.getPermission().toString().equals("rwxrwxrwx")); } + + @Test + public void testJksProviderPasswordViaConfig() throws Exception { + Configuration conf = new Configuration(); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks"; + File file = new File(tmpDir, "test.jks"); + file.delete(); + try { + conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl); + conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, + "javakeystoreprovider.password"); + KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0); + provider.createKey("key3", new byte[32], KeyProvider.options(conf)); + provider.flush(); + } catch (Exception ex) { + Assert.fail("could not create keystore with password file"); + } + KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0); + Assert.assertNotNull(provider.getCurrentKey("key3")); + + try { + conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "bar"); + KeyProviderFactory.getProviders(conf).get(0); + Assert.fail("using non existing password file, it should fail"); + } catch (IOException ex) { + //NOP + } + try { + conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "core-site.xml"); + KeyProviderFactory.getProviders(conf).get(0); + Assert.fail("using different password file, it should fail"); + } catch (IOException ex) { + //NOP + } + try { + conf.unset(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY); + KeyProviderFactory.getProviders(conf).get(0); + Assert.fail("No password file property, env not set, it should fail"); + } catch (IOException ex) { + //NOP + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password b/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password new file mode 100644 index 0000000000..1910281566 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password @@ -0,0 +1 @@ +foo \ No newline at end of file