big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17129. Validating storage keys in ABFS correctly (#2141)

Browse Source 
Contributed by Mehakmeet Singh
This commit is contained in:
Mehakmeet Singh 2020-07-16 21:59:37 +05:30 committed by GitHub
parent 1f71c4ae71
commit 4083fd57b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
View File

@ -20,7 +20,6 @@
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -219,8 +218,6 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS)
private long sasTokenRenewPeriodForStreamsInSeconds;
private Map<String, String> storageAccountKeys;
public AbfsConfiguration(final Configuration rawConfig, String accountName)
throws IllegalAccessException, InvalidConfigurationValueException, IOException {
this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders(
@ -228,7 +225,6 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName)
this.accountName = accountName;
this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false);
validateStorageAccountKeys();
Field[] fields = this.getClass().getDeclaredFields();
for (Field field : fields) {
field.setAccessible(true);
@ -740,16 +736,6 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio
}
}
void validateStorageAccountKeys() throws InvalidConfigurationValueException {
Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
this.storageAccountKeys = rawConfig.getValByRegex(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX);
for (Map.Entry<String, String> account : storageAccountKeys.entrySet()) {
validator.validate(account.getValue());
}
}
int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException {
IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class);
String value = get(validator.ConfigurationKey());

20
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException;
import org.apache.hadoop.fs.azurebfs.diagnostics.Base64StringConfigurationBasicValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,7 +44,10 @@ public String getStorageAccountKey(String accountName, Configuration rawConfig)
try {
AbfsConfiguration abfsConfig = new AbfsConfiguration(rawConfig, accountName);
key = abfsConfig.getPasswordString(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME);
} catch(IllegalAccessException | InvalidConfigurationValueException e) {
// Validating the key.
validateStorageAccountKey(key);
} catch (IllegalAccessException | InvalidConfigurationValueException e) {
throw new KeyProviderException("Failure to initialize configuration", e);
} catch(IOException ioe) {
LOG.warn("Unable to get key from credential providers. {}", ioe);
@ -51,4 +55,18 @@ public String getStorageAccountKey(String accountName, Configuration rawConfig)
return key;
}
/**
* A method to validate the storage key.
*
* @param key the key to be validated.
* @throws InvalidConfigurationValueException
*/
private void validateStorageAccountKey(String key)
throws InvalidConfigurationValueException {
Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
validator.validate(key);
}
}

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
View File

@ -30,7 +30,7 @@
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.KeyProviderException;
import org.apache.hadoop.fs.azurebfs.utils.Base64;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY;
@ -155,7 +155,7 @@ public void testGetAccountKey() throws Exception {
assertEquals(this.encodedAccountKey, accountKey);
}
@Test(expected = ConfigurationPropertyNotFoundException.class)
@Test(expected = KeyProviderException.class)
public void testGetAccountKeyWithNonExistingAccountName() throws Exception {
Configuration configuration = new Configuration();
configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);