big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.

Browse Source
This commit is contained in:
John Zhuge 2017-02-20 21:27:46 -08:00 committed by John Zhuge
parent 448ec81fd7
commit 253e3e78ab
6 changed files with 138 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -751,7 +751,20 @@ public Configuration(Configuration other) {
this.loadDefaults = other.loadDefaults;
setQuietMode(other.getQuietMode());
}
/**
* Reload existing configuration instances.
*/
public static synchronized void reloadExistingConfigurations() {
if (LOG.isDebugEnabled()) {
LOG.debug("Reloading " + REGISTRY.keySet().size()
+ " existing configurations");
}
for (Configuration conf : REGISTRY.keySet()) {
conf.reloadConfiguration();
}
}
/**
* Add a default resource. Resources are loaded in the order of the resources
* added.

38
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
View File

@ -20,6 +20,8 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configuration.DeprecationDelta;
/**
* Constants.
@ -28,25 +30,25 @@
@InterfaceStability.Evolving
public final class AdlConfKeys {
// OAuth2 Common Configuration
public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
+ ".refresh.url";
public static final String AZURE_AD_REFRESH_URL_KEY =
"fs.adl.oauth2.refresh.url";
// optional when provider type is refresh or client id.
public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
"dfs.adls.oauth2.access.token.provider";
"fs.adl.oauth2.access.token.provider";
public static final String AZURE_AD_CLIENT_ID_KEY =
"dfs.adls.oauth2.client.id";
"fs.adl.oauth2.client.id";
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
"dfs.adls.oauth2.access.token.provider.type";
"fs.adl.oauth2.access.token.provider.type";
// OAuth Refresh Token Configuration
public static final String AZURE_AD_REFRESH_TOKEN_KEY =
"dfs.adls.oauth2.refresh.token";
"fs.adl.oauth2.refresh.token";
public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
// OAuth Client Cred Token Configuration
public static final String AZURE_AD_CLIENT_SECRET_KEY =
"dfs.adls.oauth2.credential";
"fs.adl.oauth2.credential";
public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
"ClientCredential";
@ -75,7 +77,7 @@ public final class AdlConfKeys {
static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
static final String LATENCY_TRACKER_KEY =
"adl.dfs.enable.client.latency.tracker";
"adl.enable.client.latency.tracker";
static final boolean LATENCY_TRACKER_DEFAULT = true;
static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
@ -90,6 +92,26 @@ public final class AdlConfKeys {
"adl.feature.ownerandgroup.enableupn";
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
public static void addDeprecatedKeys() {
Configuration.addDeprecations(new DeprecationDelta[]{
new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type",
AZURE_AD_TOKEN_PROVIDER_TYPE_KEY),
new DeprecationDelta("dfs.adls.oauth2.client.id",
AZURE_AD_CLIENT_ID_KEY),
new DeprecationDelta("dfs.adls.oauth2.refresh.token",
AZURE_AD_REFRESH_TOKEN_KEY),
new DeprecationDelta("dfs.adls.oauth2.refresh.url",
AZURE_AD_REFRESH_URL_KEY),
new DeprecationDelta("dfs.adls.oauth2.credential",
AZURE_AD_CLIENT_SECRET_KEY),
new DeprecationDelta("dfs.adls.oauth2.access.token.provider",
AZURE_AD_TOKEN_PROVIDER_CLASS_KEY),
new DeprecationDelta("adl.dfs.enable.client.latency.tracker",
LATENCY_TRACKER_KEY)
});
Configuration.reloadExistingConfigurations();
}
private AdlConfKeys() {
}
}

4
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
View File

@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem {
private AccessTokenProvider tokenProvider;
private AzureADTokenProvider azureTokenProvider;
static {
AdlConfKeys.addDeprecatedKeys();
}
@Override
public String getScheme() {
return SCHEME;

22
hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
View File

@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml`
```xml
<property>
<name>dfs.adls.oauth2.access.token.provider.type</name>
<name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
```
@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*]
```xml
<property>
<name>dfs.adls.oauth2.client.id</name>
<name>fs.adl.oauth2.client.id</name>
<value></value>
</property>
<property>
<name>dfs.adls.oauth2.refresh.token</name>
<name>fs.adl.oauth2.refresh.token</name>
<value></value>
</property>
```
@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml`
```xml
<property>
<name>dfs.adls.oauth2.access.token.provider.type</name>
<name>fs.adl.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
</property>
<property>
<name>dfs.adls.oauth2.refresh.url</name>
<name>fs.adl.oauth2.refresh.url</name>
<value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
</property>
<property>
<name>dfs.adls.oauth2.client.id</name>
<name>fs.adl.oauth2.client.id</name>
<value>CLIENT ID FROM STEP 7 ABOVE</value>
</property>
<property>
<name>dfs.adls.oauth2.credential</name>
<name>fs.adl.oauth2.credential</name>
<value>PASSWORD FROM STEP 7 ABOVE</value>
</property>
```
@ -166,9 +166,9 @@ For additional reading on the credential provider API, see
##### Provisioning
```bash
hadoop credential create dfs.adls.oauth2.client.id -value 123
hadoop credential create fs.adl.oauth2.client.id -value 123
-provider localjceks://file/home/foo/adls.jceks
hadoop credential create dfs.adls.oauth2.refresh.token -value 123
hadoop credential create fs.adl.oauth2.refresh.token -value 123
-provider localjceks://file/home/foo/adls.jceks
```
@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```xml
<property>
<name>dfs.adls.oauth2.access.token.provider.type</name>
<name>fs.adl.oauth2.access.token.provider.type</name>
<value>RefreshToken</value>
</property>
<property>
@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
```bash
hadoop distcp
[-D dfs.adls.oauth2.access.token.provider.type=RefreshToken
[-D fs.adl.oauth2.access.token.provider.type=RefreshToken
-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
hdfs://<NameNode Hostname>:9001/user/foo/srcDir
adl://<Account Name>.azuredatalakestore.net/tgtDir/

5
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@ -121,8 +122,8 @@ public void testInvalidProviderConfigurationForType()
Assert.fail("Initialization should have failed due no token provider "
+ "configuration");
} catch (IllegalArgumentException e) {
Assert.assertTrue(
e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
GenericTestUtils.assertExceptionContains(
AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e);
}
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);

83
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.fs.adl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
@ -56,6 +58,11 @@
.TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
/**
* Validate configuration keys defined for adl storage file system instance.
*/
@ -64,18 +71,18 @@ public class TestValidateConfiguration {
@Test
public void validateConfigurationKeys() {
Assert
.assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
.assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
Assert.assertEquals("fs.adl.oauth2.access.token.provider",
AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
Assert.assertEquals("dfs.adls.oauth2.refresh.token",
Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
Assert.assertEquals("fs.adl.oauth2.refresh.token",
AZURE_AD_REFRESH_TOKEN_KEY);
Assert
.assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
.assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
Assert.assertEquals("adl.debug.override.localuserasfileowner",
ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
Assert.assertEquals("fs.adl.oauth2.access.token.provider.type",
AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
Assert.assertEquals("adl.feature.client.cache.readahead",
@ -88,7 +95,7 @@ public void validateConfigurationKeys() {
Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
Assert.assertEquals("adl.enable.client.latency.tracker",
LATENCY_TRACKER_KEY);
Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
@ -109,4 +116,66 @@ public void validateConfigurationKeys() {
Assert.assertEquals(false,
ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
}
@Test
public void testSetDeprecatedKeys() throws ClassNotFoundException {
Configuration conf = new Configuration(true);
setDeprecatedKeys(conf);
// Force AdlFileSystem static initialization to register deprecated keys.
Class.forName(AdlFileSystem.class.getName());
assertDeprecatedKeys(conf);
}
@Test
public void testLoadDeprecatedKeys()
throws IOException, ClassNotFoundException {
Configuration saveConf = new Configuration(false);
setDeprecatedKeys(saveConf);
final File testRootDir = GenericTestUtils.getTestDir();
File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml");
OutputStream out = new FileOutputStream(confXml);
saveConf.writeXml(out);
out.close();
Configuration conf = new Configuration(true);
conf.addResource(confXml.toURI().toURL());
// Trigger loading the configuration resources by getting any key.
conf.get("dummy.key");
// Force AdlFileSystem static initialization to register deprecated keys.
Class.forName(AdlFileSystem.class.getName());
assertDeprecatedKeys(conf);
}
private void setDeprecatedKeys(Configuration conf) {
conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType");
conf.set("dfs.adls.oauth2.client.id", "dummyClientId");
conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken");
conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl");
conf.set("dfs.adls.oauth2.credential", "dummyCredential");
conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
}
private void assertDeprecatedKeys(Configuration conf) {
Assert.assertEquals("dummyType",
conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY));
Assert.assertEquals("dummyClientId",
conf.get(AZURE_AD_CLIENT_ID_KEY));
Assert.assertEquals("dummyRefreshToken",
conf.get(AZURE_AD_REFRESH_TOKEN_KEY));
Assert.assertEquals("dummyRefreshUrl",
conf.get(AZURE_AD_REFRESH_URL_KEY));
Assert.assertEquals("dummyCredential",
conf.get(AZURE_AD_CLIENT_SECRET_KEY));
Assert.assertEquals("dummyClass",
conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
Assert.assertEquals("dummyTracker",
conf.get(LATENCY_TRACKER_KEY));
}
}