big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge.

Browse Source 
Signed-off-by: John Zhuge <jzhuge@apache.org>
This commit is contained in:
John Zhuge 2017-03-28 09:40:14 -07:00
parent 96cbb4fce2
commit 56e81f2a20
8 changed files with 82 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -2456,6 +2456,7 @@
<!-- Azure Data Lake File System Configurations -->
<property>
<name>fs.adl.impl</name>
<value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
@ -2465,6 +2466,68 @@
<name>fs.AbstractFileSystem.adl.impl</name>
<value>org.apache.hadoop.fs.adl.Adl</value>
</property>
<property>
<name>adl.feature.ownerandgroup.enableupn</name>
<value>false</value>
<description>
When true : User and Group in FileStatus/AclStatus response is
represented as user friendly name as per Azure AD profile.
When false (default) : User and Group in FileStatus/AclStatus
response is represented by the unique identifier from Azure AD
profile (Object ID as GUID).
For optimal performance, false is recommended.
</description>
</property>
<property>
<name>fs.adl.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
<description>
Defines Azure Active Directory OAuth2 access token provider type.
Supported types are ClientCredential, RefreshToken, and Custom.
The ClientCredential type requires property fs.adl.oauth2.client.id,
fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
The RefreshToken type requires property fs.adl.oauth2.client.id and
fs.adl.oauth2.refresh.token.
The Custom type requires property fs.adl.oauth2.access.token.provider.
</description>
</property>
<property>
<name>fs.adl.oauth2.client.id</name>
<value></value>
<description>The OAuth2 client id.</description>
</property>
<property>
<name>fs.adl.oauth2.credential</name>
<value></value>
<description>The OAuth2 access key.</description>
</property>
<property>
<name>fs.adl.oauth2.refresh.url</name>
<value></value>
<description>The OAuth2 token endpoint.</description>
</property>
<property>
<name>fs.adl.oauth2.refresh.token</name>
<value></value>
<description>The OAuth2 refresh token.</description>
</property>
<property>
<name>fs.adl.oauth2.access.token.provider</name>
<value></value>
<description>
The class name of the OAuth2 access token provider.
</description>
</property>
<!-- Azure Data Lake File System Configurations Ends Here-->
<property>

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -105,7 +105,7 @@ public void initializeMemberVariables() {
// ADL properties are in a different subtree
// - org.apache.hadoop.hdfs.web.ADLConfKeys
xmlPrefixToSkipCompare.add("adl.");
xmlPropsToSkipCompare.add("fs.adl.impl");
xmlPrefixToSkipCompare.add("fs.adl.");
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
// Azure properties are in a different class

2
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
View File

@ -40,6 +40,8 @@ public final class AdlConfKeys {
"fs.adl.oauth2.client.id";
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
"fs.adl.oauth2.access.token.provider.type";
public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT =
TokenProviderType.ClientCredential;
// OAuth Refresh Token Configuration
public static final String AZURE_AD_REFRESH_TOKEN_KEY =

3
hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
View File

@ -243,7 +243,8 @@ private AccessTokenProvider getAccessTokenProvider(Configuration config)
Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
config, AdlFileSystem.class);
TokenProviderType type = conf.getEnum(
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT);
switch (type) {
case RefreshToken:

3
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
View File

@ -28,6 +28,8 @@
import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
import com.squareup.okhttp.mockwebserver.MockWebServer;
@ -84,6 +86,7 @@ public void preTestSetup() throws IOException, URISyntaxException {
// Responses are returned in the same order that they are enqueued.
fs = new TestableAdlFileSystem();
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);

3
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
View File

@ -101,6 +101,7 @@ public void testClientCredTokenProvider()
public void testCustomCredTokenProvider()
throws URISyntaxException, IOException {
Configuration conf = new Configuration();
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
CustomMockTokenProvider.class, AzureADTokenProvider.class);
@ -115,6 +116,7 @@ public void testCustomCredTokenProvider()
public void testInvalidProviderConfigurationForType()
throws URISyntaxException, IOException {
Configuration conf = new Configuration();
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
URI uri = new URI("adl://localhost:8080");
AdlFileSystem fileSystem = new AdlFileSystem();
try {
@ -136,6 +138,7 @@ public void testInvalidProviderConfigurationForClassPath()
Configuration conf = new Configuration();
URI uri = new URI("adl://localhost:8080");
AdlFileSystem fileSystem = new AdlFileSystem();
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
"wrong.classpath.CustomMockTokenProvider");
try {

4
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
View File

@ -38,6 +38,8 @@
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
/**
* Test access token provider behaviour with custom token provider and for token
@ -89,6 +91,8 @@ public static Collection testDataForTokenProvider() {
*/
public void init() throws IOException, URISyntaxException {
Configuration configuration = new Configuration();
configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
TokenProviderType.Custom);
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
typeOfTokenProviderClass.getName());
fileSystems = new TestableAdlFileSystem[fsObjectCount];

4
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java
View File

@ -29,6 +29,8 @@
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
import static org.apache.hadoop.fs.adl.AdlConfKeys
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
/**
* This class verifies path conversion to SDK.
@ -39,6 +41,8 @@ public class TestRelativePathFormation {
public void testToRelativePath() throws URISyntaxException, IOException {
AdlFileSystem fs = new AdlFileSystem();
Configuration configuration = new Configuration();
configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
TokenProviderType.Custom);
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
"org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");