From 56e81f2a204d5e65d29222df1b2ba4d892f9c0d5 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Tue, 28 Mar 2017 09:40:14 -0700 Subject: [PATCH] HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge. Signed-off-by: John Zhuge --- .../src/main/resources/core-default.xml | 63 +++++++++++++++++++ .../conf/TestCommonConfigurationFields.java | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 2 + .../apache/hadoop/fs/adl/AdlFileSystem.java | 3 +- .../hadoop/fs/adl/AdlMockWebServer.java | 3 + .../fs/adl/TestAzureADTokenProvider.java | 3 + .../fs/adl/TestCustomTokenProvider.java | 4 ++ .../fs/adl/TestRelativePathFormation.java | 4 ++ 8 files changed, 82 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index b8f9904352..521b013654 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2456,6 +2456,7 @@ + fs.adl.impl org.apache.hadoop.fs.adl.AdlFileSystem @@ -2465,6 +2466,68 @@ fs.AbstractFileSystem.adl.impl org.apache.hadoop.fs.adl.Adl + + + adl.feature.ownerandgroup.enableupn + false + + When true : User and Group in FileStatus/AclStatus response is + represented as user friendly name as per Azure AD profile. + + When false (default) : User and Group in FileStatus/AclStatus + response is represented by the unique identifier from Azure AD + profile (Object ID as GUID). + + For optimal performance, false is recommended. + + + + + fs.adl.oauth2.access.token.provider.type + ClientCredential + + Defines Azure Active Directory OAuth2 access token provider type. + Supported types are ClientCredential, RefreshToken, and Custom. + The ClientCredential type requires property fs.adl.oauth2.client.id, + fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. + The RefreshToken type requires property fs.adl.oauth2.client.id and + fs.adl.oauth2.refresh.token. + The Custom type requires property fs.adl.oauth2.access.token.provider. + + + + + fs.adl.oauth2.client.id + + The OAuth2 client id. + + + + fs.adl.oauth2.credential + + The OAuth2 access key. + + + + fs.adl.oauth2.refresh.url + + The OAuth2 token endpoint. + + + + fs.adl.oauth2.refresh.token + + The OAuth2 refresh token. + + + + fs.adl.oauth2.access.token.provider + + + The class name of the OAuth2 access token provider. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index cbfb6d1a7c..8524973c2c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -105,7 +105,7 @@ public void initializeMemberVariables() { // ADL properties are in a different subtree // - org.apache.hadoop.hdfs.web.ADLConfKeys xmlPrefixToSkipCompare.add("adl."); - xmlPropsToSkipCompare.add("fs.adl.impl"); + xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); // Azure properties are in a different class diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index d3a5565e6d..31df22254c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -40,6 +40,8 @@ public final class AdlConfKeys { "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = "fs.adl.oauth2.access.token.provider.type"; + public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT = + TokenProviderType.ClientCredential; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0b860b3659..e63f115256 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -243,7 +243,8 @@ private AccessTokenProvider getAccessTokenProvider(Configuration config) Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( config, AdlFileSystem.class); TokenProviderType type = conf.getEnum( - AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT); switch (type) { case RefreshToken: diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java index 55c8f812c2..d843d5556b 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -84,6 +86,7 @@ public void preTestSetup() throws IOException, URISyntaxException { // Responses are returned in the same order that they are enqueued. fs = new TestableAdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 3867e74fa7..36498c6696 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -101,6 +101,7 @@ public void testClientCredTokenProvider() public void testCustomCredTokenProvider() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); @@ -115,6 +116,7 @@ public void testCustomCredTokenProvider() public void testInvalidProviderConfigurationForType() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); try { @@ -136,6 +138,7 @@ public void testInvalidProviderConfigurationForClassPath() Configuration conf = new Configuration(); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "wrong.classpath.CustomMockTokenProvider"); try { diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java index c594c6540b..737534cf01 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java @@ -38,6 +38,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * Test access token provider behaviour with custom token provider and for token @@ -89,6 +91,8 @@ public static Collection testDataForTokenProvider() { */ public void init() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, typeOfTokenProviderClass.getName()); fileSystems = new TestableAdlFileSystem[fsObjectCount]; diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java index 908f8b8919..334c372e7b 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java @@ -29,6 +29,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * This class verifies path conversion to SDK. @@ -39,6 +41,8 @@ public class TestRelativePathFormation { public void testToRelativePath() throws URISyntaxException, IOException { AdlFileSystem fs = new AdlFileSystem(); Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");