diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java index ce99d795f3..b16960c178 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; /** * A factory to create a list of KeyProvider based on the path given in a @@ -39,7 +40,7 @@ @InterfaceStability.Unstable public abstract class KeyProviderFactory { public static final String KEY_PROVIDER_PATH = - "hadoop.security.key.provider.path"; + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; public abstract KeyProvider createProvider(URI providerName, Configuration conf diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 0a3afb781f..b5b107c2b3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -628,6 +628,14 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS = "hadoop.security.impersonation.provider.class"; + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_SECURITY_KEY_PROVIDER_PATH = + "hadoop.security.key.provider.path"; + // /** * @see diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index fda8c9bc81..5b8d49d313 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2037,6 +2037,14 @@ + + hadoop.security.key.provider.path + + The KeyProvider to use when managing zone keys, and interacting with + encryption keys when reading and writing to an encryption zone. + + + fs.har.impl.disable.cache true diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index adebe6e981..94ed68e36a 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -28,6 +28,7 @@ The following table lists the configuration property names that are deprecated i | dfs.data.dir | dfs.datanode.data.dir | | dfs.datanode.max.xcievers | dfs.datanode.max.transfer.threads | | dfs.df.interval | fs.df.interval | +| dfs.encryption.key.provider.uri | hadoop.security.key.provider.path | | dfs.federation.nameservice.id | dfs.nameservice.id | | dfs.federation.nameservices | dfs.nameservices | | dfs.http.address | dfs.namenode.http-address | diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index 889dbaf177..0c6d0b22da 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -37,10 +37,10 @@ KMS Client Configuration The KMS client `KeyProvider` uses the **kms** scheme, and the embedded URL must be the URL of the KMS. For example, for a KMS running on `http://localhost:9600/kms`, the KeyProvider URI is `kms://http@localhost:9600/kms`. And, for a KMS running on `https://localhost:9600/kms`, the KeyProvider URI is `kms://https@localhost:9600/kms` The following is an example to configure HDFS NameNode as a KMS client in -`hdfs-site.xml`: +`core-site.xml`: - dfs.encryption.key.provider.uri + hadoop.security.key.provider.path kms://http@localhost:9600/kms The KeyProvider to use when interacting with encryption keys used @@ -664,15 +664,15 @@ is to use LoadBalancingKMSClientProvider. Using this approach, a KMS client (for example, a HDFS NameNode) is aware of multiple KMS instances, and it sends requests to them in a round-robin fashion. LoadBalancingKMSClientProvider is implicitly used when more than one URI is specified in -`dfs.encryption.key.provider.uri`. +`hadoop.security.key.provider.path`. -The following example in `hdfs-site.xml` configures two KMS +The following example in `core-site.xml` configures two KMS instances, `kms01.example.com` and `kms02.example.com`. The hostnames are separated by semi-colons, and all KMS instances must run on the same port. - dfs.encryption.key.provider.uri + hadoop.security.key.provider.path kms://https@kms01.example.com;kms02.example.com:9600/kms The KeyProvider to use when interacting with encryption keys used diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index c5115961ee..b93632aaec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -526,7 +526,7 @@ public static ReconfigurationProtocol createReconfigurationProtocolProxy( } private static String keyProviderUriKeyName = - HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI; + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; /** * Set the key provider uri configuration key name for creating key providers. @@ -616,16 +616,17 @@ public static int getSmallBufferSize(Configuration conf) { } /** - * Probe for HDFS Encryption being enabled; this uses the value of - * the option {@link HdfsClientConfigKeys#DFS_ENCRYPTION_KEY_PROVIDER_URI}, - * returning true if that property contains a non-empty, non-whitespace + * Probe for HDFS Encryption being enabled; this uses the value of the option + * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_KEY_PROVIDER_PATH} + * , returning true if that property contains a non-empty, non-whitespace * string. * @param conf configuration to probe * @return true if encryption is considered enabled. */ public static boolean isHDFSEncryptionEnabled(Configuration conf) { - return !conf.getTrimmed( - HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "").isEmpty(); + return !(conf.getTrimmed( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "") + .isEmpty()); } public static InetSocketAddress getNNAddress(String address) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java index 580e7f9b59..3f6eae0e9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys; @@ -141,6 +142,8 @@ private static void addDeprecatedKeys() { HdfsClientConfigKeys.DFS_NAMESERVICES), new DeprecationDelta("dfs.federation.nameservice.id", DeprecatedKeys.DFS_NAMESERVICE_ID), + new DeprecationDelta("dfs.encryption.key.provider.uri", + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH), }); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java index 0b0e00643b..3491010aa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; @@ -86,11 +86,11 @@ public KeyProvider call() throws Exception { private URI createKeyProviderURI(Configuration conf) { final String providerUriStr = conf.getTrimmed( - HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, ""); + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""); // No provider set in conf if (providerUriStr.isEmpty()) { LOG.error("Could not find uri with key [" - + HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI + + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH + "] to create a keyProvider !!"); return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 3ea42c38b3..642d4c826b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -139,7 +139,6 @@ public interface HdfsClientConfigKeys { "dfs.datanode.kerberos.principal"; String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes"; long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB - String DFS_ENCRYPTION_KEY_PROVIDER_URI = "dfs.encryption.key.provider.uri"; String DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY = "dfs.encrypt.data.transfer.cipher.suites"; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java index 5695285961..0e701f712e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; @@ -158,7 +159,8 @@ private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) thr FileSystemTestHelper helper = new FileSystemTestHelper(); final String jceksPath = JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(helper.getTestRootDir(), "test.jks").toUri(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, jceksPath); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + jceksPath); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); builder.numDataNodes(2); MiniDFSCluster miniHdfs = builder.build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java index 811aedae25..0fabb300db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java @@ -30,10 +30,10 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -135,7 +135,7 @@ public static void setup() throws Exception { String testRoot = fsHelper.getTestRootDir(); testRootDir = new File(testRoot).getAbsoluteFile(); final Path jksPath = new Path(testRootDir.toString(), "test.jks"); - config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); ProxyUsers.refreshSuperUserGroupsConfiguration(config); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index caf6b6078d..3532d25764 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -804,8 +804,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { HdfsClientConfigKeys.DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY; public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100; public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses"; - public static final String DFS_ENCRYPTION_KEY_PROVIDER_URI = - HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI; public static final String DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_KEY = "dfs.namenode.edekcacheloader.interval.ms"; public static final int DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_DEFAULT = 1000; public static final String DFS_NAMENODE_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY = "dfs.namenode.edekcacheloader.initial.delay.ms"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 29c9ef25cf..ebaefde262 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2756,14 +2756,6 @@ - - dfs.encryption.key.provider.uri - - The KeyProvider to use when interacting with encryption keys used - when reading and writing to an encryption zone. - - - dfs.storage.policy.enabled true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index 6d0e52c891..ee98df8308 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -117,7 +117,7 @@ Once a KMS has been set up and the NameNode and HDFS clients have been correctly ### Configuring the cluster KeyProvider -#### dfs.encryption.key.provider.uri +#### hadoop.security.key.provider.path The KeyProvider to use when interacting with encryption keys used when reading and writing to an encryption zone. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java index 7c6aaeb9ed..99a7c2ac26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java @@ -35,7 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -66,7 +66,7 @@ public void setUp() throws Exception { tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString()); final Path jksPath = new Path(tmpDir.toString(), "test.jks"); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java index 2b515d00cb..3ff705588a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java @@ -36,6 +36,7 @@ import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration; import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider; import org.apache.hadoop.crypto.key.kms.server.MiniKMS; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystemTestHelper; @@ -190,7 +191,7 @@ private void setup(Configuration conf, boolean resetKms, boolean resetDfs) "keyadmin,hdfs,user"); conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER + "." + realUser + ".hosts", "*"); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, getKeyProviderURI()); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index b7db60148f..7257bbd793 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -59,6 +59,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -1031,16 +1032,19 @@ public void testGetNNServiceRpcAddressesForNsIds() throws IOException { @Test public void testEncryptionProbe() throws Throwable { Configuration conf = new Configuration(false); - conf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI); + conf.unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); assertFalse("encryption enabled on no provider key", DFSUtilClient.isHDFSEncryptionEnabled(conf)); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, ""); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + ""); assertFalse("encryption enabled on empty provider key", DFSUtilClient.isHDFSEncryptionEnabled(conf)); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "\n\t\n"); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + "\n\t\n"); assertFalse("encryption enabled on whitespace provider key", DFSUtilClient.isHDFSEncryptionEnabled(conf)); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "http://hadoop.apache.org"); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + "http://hadoop.apache.org"); assertTrue("encryption disabled on valid provider key", DFSUtilClient.isHDFSEncryptionEnabled(conf)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 53c12ec40e..b634dd26bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -150,7 +150,8 @@ public void setup() throws Exception { // Set up java key store String testRoot = fsHelper.getTestRootDir(); testRootDir = new File(testRoot).getAbsoluteFile(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + getKeyProviderURI()); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, @@ -845,9 +846,9 @@ public void testVersionAndSuiteNegotiation() throws Exception { // Check KeyProvider state // Flushing the KP on the NN, since it caches, and init a test one cluster.getNamesystem().getProvider().flush(); - KeyProvider provider = KeyProviderFactory - .get(new URI(conf.getTrimmed(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI)), - conf); + KeyProvider provider = KeyProviderFactory.get(new URI(conf.getTrimmed( + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH)), + conf); List keys = provider.getKeys(); assertEquals("Expected NN to have created one key per zone", 1, keys.size()); @@ -931,7 +932,8 @@ private static void testCipherSuiteNegotiation(DistributedFileSystem fs, public void testCreateEZWithNoProvider() throws Exception { // Unset the key provider and make sure EZ ops don't work final Configuration clusterConf = cluster.getConfiguration(0); - clusterConf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI); + clusterConf + .unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); cluster.restartNameNode(true); cluster.waitActive(); final Path zone1 = new Path("/zone1"); @@ -943,8 +945,9 @@ public void testCreateEZWithNoProvider() throws Exception { assertExceptionContains("since no key provider is available", e); } final Path jksPath = new Path(testRootDir.toString(), "test.jks"); - clusterConf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, - JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri() + clusterConf + .set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri() ); // Try listing EZs as well assertNumZones(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java index f3a26400af..bb8b79b723 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; @@ -62,7 +63,7 @@ public void setupCluster() throws Exception { fsHelper = new FileSystemTestHelper(); String testRoot = fsHelper.getTestRootDir(); testRootDir = new File(testRoot).getAbsoluteFile(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(testRootDir.toString(), "test.jks").toUri() ); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java index 4cbe871df5..0868b5441e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java @@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderFactory; -import org.apache.hadoop.crypto.key.kms.KMSClientProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.junit.Assert; import org.junit.Test; @@ -94,26 +94,26 @@ public KeyProvider createProvider(URI providerName, Configuration conf) public void testCache() throws Exception { KeyProviderCache kpCache = new KeyProviderCache(10000); Configuration conf = new Configuration(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://foo:bar@test_provider1"); KeyProvider keyProvider1 = kpCache.get(conf); Assert.assertNotNull("Returned Key Provider is null !!", keyProvider1); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://foo:bar@test_provider1"); KeyProvider keyProvider2 = kpCache.get(conf); Assert.assertTrue("Different KeyProviders returned !!", keyProvider1 == keyProvider2); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://test_provider3"); KeyProvider keyProvider3 = kpCache.get(conf); Assert.assertFalse("Same KeyProviders returned !!", keyProvider1 == keyProvider3); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "dummy://hello:there@test_provider1"); KeyProvider keyProvider4 = kpCache.get(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java index c09d34644f..5416739304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestWrapper; import org.apache.hadoop.fs.FileStatus; @@ -77,7 +78,7 @@ public void setup() throws Exception { String testRoot = fsHelper.getTestRootDir(); File testRootDir = new File(testRoot).getAbsoluteFile(); final Path jksPath = new Path(testRootDir.toString(), "test.jks"); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri() ); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java index b85c069975..96e39ba0bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java @@ -49,6 +49,7 @@ import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration; import org.apache.hadoop.crypto.key.kms.server.MiniKMS; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestWrapper; import org.apache.hadoop.fs.FileUtil; @@ -237,8 +238,9 @@ public static void destroy() { @Before public void setup() throws Exception { // Start MiniDFS Cluster - baseConf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, - getKeyProviderURI()); + baseConf + .set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + getKeyProviderURI()); baseConf.setBoolean(DFSConfigKeys .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java index 8847c91e20..13fc985180 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -85,7 +86,8 @@ public void setup() throws Exception { // Set up java key store String testRoot = fsHelper.getTestRootDir(); testRootDir = new File(testRoot).getAbsoluteFile(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + getKeyProviderURI()); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 9e95650d99..90d61eeee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode.metrics; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileSystemTestWrapper; import org.apache.hadoop.fs.permission.FsPermission; @@ -642,7 +643,7 @@ public void testGenerateEDEKTime() throws IOException, // Set up java key store String testRoot = fsHelper.getTestRootDir(); File testRootDir = new File(testRoot).getAbsoluteFile(); - conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(testRootDir.toString(), "test.jks").toUri()); conf.setBoolean(DFSConfigKeys