big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17053. ABFS: Fix Account-specific OAuth config setting parsing

Browse Source 
Contributed by Sneha Vijayarajan
This commit is contained in:
Sneha Vijayarajan 2020-05-27 13:56:09 -07:00 committed by GitHub
parent 53b993e604
commit 4c5cd751e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 265 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View File

@ -325,31 +325,91 @@ public String getPasswordString(String key) throws IOException {
}
/**
* Returns the account-specific Class if it exists, then looks for an
* account-agnostic value, and finally tries the default value.
* Returns account-specific token provider class if it exists, else checks if
* an account-agnostic setting is present for token provider class if AuthType
* matches with authType passed.
* @param authType AuthType effective on the account
* @param name Account-agnostic configuration key
* @param defaultValue Class returned if none is configured
* @param xface Interface shared by all possible values
* @param <U> Interface class type
* @return Highest-precedence Class object that was found
*/
public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
public <U> Class<? extends U> getTokenProviderClass(AuthType authType,
String name,
Class<? extends U> defaultValue,
Class<U> xface) {
Class<?> tokenProviderClass = getAccountSpecificClass(name, defaultValue,
xface);
// If there is none set specific for account
// fall back to generic setting if Auth Type matches
if ((tokenProviderClass == null)
&& (authType == getAccountAgnosticEnum(
FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) {
tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface);
}
return (tokenProviderClass == null)
? null
: tokenProviderClass.asSubclass(xface);
}
/**
* Returns the account-specific class if it exists, else returns default value.
* @param name Account-agnostic configuration key
* @param defaultValue Class returned if none is configured
* @param xface Interface shared by all possible values
* @param <U> Interface class type
* @return Account specific Class object that was found
*/
public <U> Class<? extends U> getAccountSpecificClass(String name,
Class<? extends U> defaultValue,
Class<U> xface) {
return rawConfig.getClass(accountConf(name),
rawConfig.getClass(name, defaultValue, xface),
defaultValue,
xface);
}
/**
* Returns the account-specific password in string form if it exists, then
* Returns account-agnostic Class if it exists, else returns the default value.
* @param name Account-agnostic configuration key
* @param defaultValue Class returned if none is configured
* @param xface Interface shared by all possible values
* @param <U> Interface class type
* @return Account-Agnostic Class object that was found
*/
public <U> Class<? extends U> getAccountAgnosticClass(String name,
Class<? extends U> defaultValue,
Class<U> xface) {
return rawConfig.getClass(name, defaultValue, xface);
}
/**
* Returns the account-specific enum value if it exists, then
* looks for an account-agnostic value.
* @param name Account-agnostic configuration key
* @param defaultValue Value returned if none is configured
* @return value in String form if one exists, else null
* @param <T> Enum type
* @return enum value if one exists, else null
*/
public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
return rawConfig.getEnum(accountConf(name),
rawConfig.getEnum(name, defaultValue));
}
/**
* Returns the account-agnostic enum value if it exists, else
* return default.
* @param name Account-agnostic configuration key
* @param defaultValue Value returned if none is configured
* @param <T> Enum type
* @return enum value if one exists, else null
*/
public <T extends Enum<T>> T getAccountAgnosticEnum(String name, T defaultValue) {
return rawConfig.getEnum(name, defaultValue);
}
/**
* Unsets parameter in the underlying Configuration object.
* Provided only as a convenience; does not add any account logic.
@ -560,8 +620,10 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio
if (authType == AuthType.OAuth) {
try {
Class<? extends AccessTokenProvider> tokenProviderClass =
getClass(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null,
AccessTokenProvider.class);
getTokenProviderClass(authType,
FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null,
AccessTokenProvider.class);
AccessTokenProvider tokenProvider = null;
if (tokenProviderClass == ClientCredsTokenProvider.class) {
String authEndpoint = getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT);
@ -604,14 +666,17 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio
} catch(IllegalArgumentException e) {
throw e;
} catch (Exception e) {
throw new TokenAccessProviderException("Unable to load key provider class.", e);
throw new TokenAccessProviderException("Unable to load OAuth token provider class.", e);
}
} else if (authType == AuthType.Custom) {
try {
String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
Class<? extends CustomTokenProviderAdaptee> customTokenProviderClass =
getClass(configKey, null, CustomTokenProviderAdaptee.class);
Class<? extends CustomTokenProviderAdaptee> customTokenProviderClass
= getTokenProviderClass(authType, configKey, null,
CustomTokenProviderAdaptee.class);
if (customTokenProviderClass == null) {
throw new IllegalArgumentException(
String.format("The configuration value for \"%s\" is invalid.", configKey));
@ -647,7 +712,9 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio
try {
String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
Class<? extends SASTokenProvider> sasTokenProviderClass =
getClass(configKey, null, SASTokenProvider.class);
getTokenProviderClass(authType, configKey, null,
SASTokenProvider.class);
Preconditions.checkArgument(sasTokenProviderClass != null,
String.format("The configuration value for \"%s\" is invalid.", configKey));

193
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java
View File

@ -20,13 +20,24 @@
import java.io.IOException;
import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider;
import org.apache.hadoop.fs.azurebfs.oauth2.CustomTokenProviderAdapter;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.junit.Test;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
/**
* Tests correct precedence of various configurations that might be returned.
@ -40,6 +51,14 @@
* that do allow default values (all others) follow another form.
*/
public class TestAccountConfiguration {
private static final String TEST_OAUTH_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider";
private static final String TEST_CUSTOM_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider";
private static final String TEST_SAS_PROVIDER_CLASS_CONFIG_1 = "org.apache.hadoop.fs.azurebfs.extensions.MockErrorSASTokenProvider";
private static final String TEST_SAS_PROVIDER_CLASS_CONFIG_2 = "org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider";
private static final String TEST_OAUTH_ENDPOINT = "oauthEndpoint";
private static final String TEST_CLIENT_ID = "clientId";
private static final String TEST_CLIENT_SECRET = "clientSecret";
@Test
public void testStringPrecedence()
@ -248,7 +267,7 @@ private class GetClassImpl1 implements GetClassInterface {
}
@Test
public void testClassPrecedence()
public void testClass()
throws IllegalAccessException, IOException, InvalidConfigurationValueException {
final String accountName = "account";
@ -264,22 +283,182 @@ public void testClassPrecedence()
conf.setClass(globalKey, class0, xface);
assertEquals("Default value returned even though account-agnostic config was set",
abfsConf.getClass(globalKey, class1, xface), class0);
abfsConf.getAccountAgnosticClass(globalKey, class1, xface), class0);
conf.unset(globalKey);
assertEquals("Default value not returned even though config was unset",
abfsConf.getClass(globalKey, class1, xface), class1);
abfsConf.getAccountAgnosticClass(globalKey, class1, xface), class1);
conf.setClass(accountKey, class0, xface);
assertEquals("Default value returned even though account-specific config was set",
abfsConf.getClass(globalKey, class1, xface), class0);
abfsConf.getAccountSpecificClass(globalKey, class1, xface), class0);
conf.unset(accountKey);
assertEquals("Default value not returned even though config was unset",
abfsConf.getClass(globalKey, class1, xface), class1);
abfsConf.getAccountSpecificClass(globalKey, class1, xface), class1);
conf.setClass(accountKey, class1, xface);
conf.setClass(globalKey, class0, xface);
assertEquals("Account-agnostic or default value returned even though account-specific config was set",
abfsConf.getClass(globalKey, class0, xface), class1);
abfsConf.getAccountSpecificClass(globalKey, class0, xface), class1);
}
@Test
public void testSASProviderPrecedence()
throws IOException, IllegalAccessException {
final String accountName = "account";
final Configuration conf = new Configuration();
final AbfsConfiguration abfsConf = new AbfsConfiguration(conf, accountName);
// AccountSpecific: SAS with provider set as SAS_Provider_1
abfsConf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + "." + accountName,
"SAS");
abfsConf.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE + "." + accountName,
TEST_SAS_PROVIDER_CLASS_CONFIG_1);
// Global: SAS with provider set as SAS_Provider_2
abfsConf.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME,
AuthType.SAS.toString());
abfsConf.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE,
TEST_SAS_PROVIDER_CLASS_CONFIG_2);
Assertions.assertThat(
abfsConf.getSASTokenProvider().getClass().getName())
.describedAs(
"Account-specific SAS token provider should be in effect.")
.isEqualTo(TEST_SAS_PROVIDER_CLASS_CONFIG_1);
}
@Test
public void testAccessTokenProviderPrecedence()
throws IllegalAccessException, IOException {
final String accountName = "account";
final Configuration conf = new Configuration();
final AbfsConfiguration abfsConf = new AbfsConfiguration(conf, accountName);
// Global: Custom , AccountSpecific: OAuth
testGlobalAndAccountOAuthPrecedence(abfsConf, AuthType.Custom,
AuthType.OAuth);
// Global: OAuth , AccountSpecific: Custom
testGlobalAndAccountOAuthPrecedence(abfsConf, AuthType.OAuth,
AuthType.Custom);
// Global: (non-oAuth) SAS , AccountSpecific: Custom
testGlobalAndAccountOAuthPrecedence(abfsConf, AuthType.SAS,
AuthType.Custom);
// Global: Custom , AccountSpecific: -
testGlobalAndAccountOAuthPrecedence(abfsConf, AuthType.Custom, null);
// Global: OAuth , AccountSpecific: -
testGlobalAndAccountOAuthPrecedence(abfsConf, AuthType.OAuth, null);
// Global: - , AccountSpecific: Custom
testGlobalAndAccountOAuthPrecedence(abfsConf, null, AuthType.Custom);
// Global: - , AccountSpecific: OAuth
testGlobalAndAccountOAuthPrecedence(abfsConf, null, AuthType.OAuth);
}
public void testGlobalAndAccountOAuthPrecedence(AbfsConfiguration abfsConf,
AuthType globalAuthType,
AuthType accountSpecificAuthType)
throws IOException {
if (globalAuthType == null) {
unsetAuthConfig(abfsConf, false);
} else {
setAuthConfig(abfsConf, false, globalAuthType);
}
if (accountSpecificAuthType == null) {
unsetAuthConfig(abfsConf, true);
} else {
setAuthConfig(abfsConf, true, accountSpecificAuthType);
}
// If account specific AuthType is present, precedence is always for it.
AuthType expectedEffectiveAuthType;
if (accountSpecificAuthType != null) {
expectedEffectiveAuthType = accountSpecificAuthType;
} else {
expectedEffectiveAuthType = globalAuthType;
}
Class<?> expectedEffectiveTokenProviderClassType =
(expectedEffectiveAuthType == AuthType.OAuth)
? ClientCredsTokenProvider.class
: CustomTokenProviderAdapter.class;
Assertions.assertThat(
abfsConf.getTokenProvider().getClass().getTypeName())
.describedAs(
"Account-specific settings takes precendence to global"
+ " settings. In absence of Account settings, global settings "
+ "should take effect.")
.isEqualTo(expectedEffectiveTokenProviderClassType.getTypeName());
unsetAuthConfig(abfsConf, false);
unsetAuthConfig(abfsConf, true);
}
public void setAuthConfig(AbfsConfiguration abfsConf,
boolean isAccountSetting,
AuthType authType) {
final String accountNameSuffix = "." + abfsConf.getAccountName();
String authKey = FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME
+ (isAccountSetting ? accountNameSuffix : "");
String providerClassKey = "";
String providerClassValue = "";
switch (authType) {
case OAuth:
providerClassKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME
+ (isAccountSetting ? accountNameSuffix : "");
providerClassValue = TEST_OAUTH_PROVIDER_CLASS_CONFIG;
abfsConf.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT
+ ((isAccountSetting) ? accountNameSuffix : ""),
TEST_OAUTH_ENDPOINT);
abfsConf.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID
+ ((isAccountSetting) ? accountNameSuffix : ""),
TEST_CLIENT_ID);
abfsConf.set(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET
+ ((isAccountSetting) ? accountNameSuffix : ""),
TEST_CLIENT_SECRET);
break;
case Custom:
providerClassKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME
+ (isAccountSetting ? accountNameSuffix : "");
providerClassValue = TEST_CUSTOM_PROVIDER_CLASS_CONFIG;
break;
case SAS:
providerClassKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE
+ (isAccountSetting ? accountNameSuffix : "");
providerClassValue = TEST_SAS_PROVIDER_CLASS_CONFIG_1;
break;
default: // set nothing
}
abfsConf.set(authKey, authType.toString());
abfsConf.set(providerClassKey, providerClassValue);
}
private void unsetAuthConfig(AbfsConfiguration abfsConf, boolean isAccountSettings) {
String accountNameSuffix =
isAccountSettings ? ("." + abfsConf.getAccountName()) : "";
abfsConf.unset(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountNameSuffix);
abfsConf.unset(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountNameSuffix);
abfsConf.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE + accountNameSuffix);
abfsConf.unset(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT + accountNameSuffix);
abfsConf.unset(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID + accountNameSuffix);
abfsConf.unset(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET + accountNameSuffix);
}
}