From df57c6c3b12117788b78c30b9d0703c5e9d88458 Mon Sep 17 00:00:00 2001 From: Thomas Marquardt Date: Wed, 29 Aug 2018 05:59:44 +0000 Subject: [PATCH] HADOOP-15692. ABFS: extensible support for custom oauth. Contributed by Junhua Gu and Rajeev Bansal. --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 63 +++++++----- .../fs/azurebfs/AzureBlobFileSystem.java | 28 ++++++ .../azurebfs/constants/ConfigurationKeys.java | 3 + .../constants/FileSystemConfigurations.java | 1 + .../CustomDelegationTokenManager.java | 66 +++++++++++++ .../CustomTokenProviderAdaptee.java | 8 +- .../fs/azurebfs/extensions/package-info.java | 21 ++++ .../oauth2/CustomTokenProviderAdapter.java | 1 + .../AbfsDelegationTokenIdentifier.java | 49 ++++++++++ .../security/AbfsDelegationTokenManager.java | 88 +++++++++++++++++ .../azurebfs/security/AbfsTokenRenewer.java | 96 +++++++++++++++++++ .../fs/azurebfs/security/package-info.java | 23 +++++ ...ache.hadoop.security.token.TokenIdentifier | 1 + ....apache.hadoop.security.token.TokenRenewer | 1 + .../ITestAzureBlobFileSystemFileStatus.java | 3 +- 15 files changed, 421 insertions(+), 31 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomDelegationTokenManager.java rename hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{oauth2 => extensions}/CustomTokenProviderAdaptee.java (94%) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenIdentifier.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsTokenRenewer.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/package-info.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index f26f56204c..069f17a1a7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation; import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation; @@ -43,13 +42,14 @@ import org.apache.hadoop.fs.azurebfs.diagnostics.IntegerConfigurationBasicValidator; import org.apache.hadoop.fs.azurebfs.diagnostics.LongConfigurationBasicValidator; import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator; +import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee; import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; -import org.apache.hadoop.fs.azurebfs.oauth2.CustomTokenProviderAdaptee; import org.apache.hadoop.fs.azurebfs.oauth2.CustomTokenProviderAdapter; import org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.RefreshTokenBasedTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.UserPasswordTokenProvider; +import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.services.KeyProvider; import org.apache.hadoop.fs.azurebfs.services.SimpleKeyProvider; @@ -57,7 +57,7 @@ import org.apache.hadoop.util.ReflectionUtils; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_SSL_CHANNEL_MODE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*; /** * Configuration for Azure Blob FileSystem. @@ -69,83 +69,86 @@ public class AbfsConfiguration{ private final boolean isSecure; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, - MinValue = FileSystemConfigurations.MIN_BUFFER_SIZE, - MaxValue = FileSystemConfigurations.MAX_BUFFER_SIZE, - DefaultValue = FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE) + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_WRITE_BUFFER_SIZE) private int writeBufferSize; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE, - MinValue = FileSystemConfigurations.MIN_BUFFER_SIZE, - MaxValue = FileSystemConfigurations.MAX_BUFFER_SIZE, - DefaultValue = FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE) + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_BUFFER_SIZE) private int readBufferSize; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MIN_BACKOFF_INTERVAL, - DefaultValue = FileSystemConfigurations.DEFAULT_MIN_BACKOFF_INTERVAL) + DefaultValue = DEFAULT_MIN_BACKOFF_INTERVAL) private int minBackoffInterval; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_BACKOFF_INTERVAL, - DefaultValue = FileSystemConfigurations.DEFAULT_MAX_BACKOFF_INTERVAL) + DefaultValue = DEFAULT_MAX_BACKOFF_INTERVAL) private int maxBackoffInterval; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BACKOFF_INTERVAL, - DefaultValue = FileSystemConfigurations.DEFAULT_BACKOFF_INTERVAL) + DefaultValue = DEFAULT_BACKOFF_INTERVAL) private int backoffInterval; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_IO_RETRIES, MinValue = 0, - DefaultValue = FileSystemConfigurations.DEFAULT_MAX_RETRY_ATTEMPTS) + DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) private int maxIoRetries; @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, MinValue = 0, - MaxValue = FileSystemConfigurations.MAX_AZURE_BLOCK_SIZE, - DefaultValue = FileSystemConfigurations.MAX_AZURE_BLOCK_SIZE) + MaxValue = MAX_AZURE_BLOCK_SIZE, + DefaultValue = MAX_AZURE_BLOCK_SIZE) private long azureBlockSize; @StringConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, - DefaultValue = FileSystemConfigurations.AZURE_BLOCK_LOCATION_HOST_DEFAULT) + DefaultValue = AZURE_BLOCK_LOCATION_HOST_DEFAULT) private String azureBlockLocationHost; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_OUT, MinValue = 1, - DefaultValue = FileSystemConfigurations.MAX_CONCURRENT_WRITE_THREADS) + DefaultValue = MAX_CONCURRENT_WRITE_THREADS) private int maxConcurrentWriteThreads; @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_IN, MinValue = 1, - DefaultValue = FileSystemConfigurations.MAX_CONCURRENT_READ_THREADS) + DefaultValue = MAX_CONCURRENT_READ_THREADS) private int maxConcurrentReadThreads; @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_TOLERATE_CONCURRENT_APPEND, - DefaultValue = FileSystemConfigurations.DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) + DefaultValue = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) private boolean tolerateOobAppends; @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ATOMIC_RENAME_KEY, - DefaultValue = FileSystemConfigurations.DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) + DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) private String azureAtomicDirs; @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - DefaultValue = FileSystemConfigurations.DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) + DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) private boolean createRemoteFileSystemDuringInitialization; @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION, - DefaultValue = FileSystemConfigurations.DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) + DefaultValue = DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) private boolean skipUserGroupMetadataDuringInitialization; - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_QUEUE_DEPTH, - DefaultValue = FileSystemConfigurations.DEFAULT_READ_AHEAD_QUEUE_DEPTH) + DefaultValue = DEFAULT_READ_AHEAD_QUEUE_DEPTH) private int readAheadQueueDepth; @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_FLUSH, - DefaultValue = FileSystemConfigurations.DEFAULT_ENABLE_FLUSH) + DefaultValue = DEFAULT_ENABLE_FLUSH) private boolean enableFlush; @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY, - DefaultValue = "") + DefaultValue = "") private String userAgentId; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN, + DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN) + private boolean enableDelegationToken; + private Map storageAccountKeys; public AbfsConfiguration(final Configuration configuration) throws IllegalAccessException, InvalidConfigurationValueException { @@ -292,6 +295,14 @@ public AuthType getAuthType(final String accountName) { return configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); } + public boolean isDelegationTokenManagerEnabled() { + return enableDelegationToken; + } + + public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { + return new AbfsDelegationTokenManager(configuration); + } + public AccessTokenProvider getTokenProvider(final String accountName) throws TokenAccessProviderException { AuthType authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountName, AuthType.SharedKey); if (authType == AuthType.OAuth) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 6bec7cb2a3..4bde9d80eb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -61,9 +61,11 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; @@ -82,6 +84,9 @@ public class AzureBlobFileSystem extends FileSystem { private AzureBlobFileSystemStore abfsStore; private boolean isClosed; + private boolean delegationTokenEnabled = false; + private AbfsDelegationTokenManager delegationTokenManager; + @Override public void initialize(URI uri, Configuration configuration) throws IOException { @@ -112,6 +117,15 @@ public void initialize(URI uri, Configuration configuration) //Provide a default group name this.primaryUserGroup = this.user; } + + if (UserGroupInformation.isSecurityEnabled()) { + this.delegationTokenEnabled = abfsStore.getAbfsConfiguration().isDelegationTokenManagerEnabled(); + + if(this.delegationTokenEnabled) { + LOG.debug("Initializing DelegationTokenManager for {}", uri); + this.delegationTokenManager = abfsStore.getAbfsConfiguration().getDelegationTokenManager(); + } + } } @Override @@ -815,6 +829,20 @@ private Throwable getRootCause(Throwable throwable) { return result; } + /** + * Get a delegation token from remote service endpoint if + * 'fs.azure.enable.kerberos.support' is set to 'true', and + * 'fs.azure.enable.delegation.token' is set to 'true'. + * @param renewer the account name that is allowed to renew the token. + * @return delegation token + * @throws IOException thrown when getting the current user. + */ + @Override + public synchronized Token getDelegationToken(final String renewer) throws IOException { + return this.delegationTokenEnabled ? this.delegationTokenManager.getDelegationToken(renewer) + : super.getDelegationToken(renewer); + } + @VisibleForTesting FileSystem.Statistics getFsStatistics() { return this.statistics; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index ffdf700b78..8ad0dc6182 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -79,5 +79,8 @@ public final class ConfigurationKeys { /** Prefix for oauth refresh token: {@value}. */ public static final String FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN = "fs.azure.account.oauth2.refresh.token."; + public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; + public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type"; + private ConfigurationKeys() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 5b92dddf73..a921faf8d5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -61,5 +61,6 @@ public final class FileSystemConfigurations { public static final SSLSocketFactoryEx.SSLChannelMode DEFAULT_FS_AZURE_SSL_CHANNEL_MODE = SSLSocketFactoryEx.SSLChannelMode.Default; + public static final boolean DEFAULT_ENABLE_DELEGATION_TOKEN = false; private FileSystemConfigurations() {} } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomDelegationTokenManager.java new file mode 100644 index 0000000000..f5716e089c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomDelegationTokenManager.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.fs.azurebfs.extensions; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; +import org.apache.hadoop.security.token.Token; + +/** + * Interface for Managing the Delegation tokens. + */ +public interface CustomDelegationTokenManager { + + /** + * Initialize with supported configuration. This method is invoked when the + * (URI, Configuration)} method is invoked. + * + * @param configuration Configuration object + * @throws IOException if instance can not be configured. + */ + void initialize(Configuration configuration) + throws IOException; + + /** + * Get Delegation token. + * @param renewer delegation token renewer + * @return delegation token + * @throws IOException when error in getting the delegation token + */ + Token getDelegationToken(String renewer) + throws IOException; + + /** + * Renew the delegation token. + * @param token delegation token. + * @return renewed time. + * @throws IOException when error in renewing the delegation token + */ + long renewDelegationToken(Token token) throws IOException; + + /** + * Cancel the delegation token. + * @param token delegation token. + * @throws IOException when error in cancelling the delegation token. + */ + void cancelDelegationToken(Token token) throws IOException; +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdaptee.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomTokenProviderAdaptee.java similarity index 94% rename from hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdaptee.java rename to hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomTokenProviderAdaptee.java index 7366a8d795..889d506c4a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdaptee.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/CustomTokenProviderAdaptee.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.fs.azurebfs.oauth2; +package org.apache.hadoop.fs.azurebfs.extensions; import java.io.IOException; import java.util.Date; @@ -47,8 +47,8 @@ public interface CustomTokenProviderAdaptee { * @param accountName Account Name * @throws IOException if instance can not be configured. */ - void initialize(Configuration configuration, String accountName) - throws IOException; + void initialize(Configuration configuration, final String accountName) + throws IOException; /** * Obtain the access token that should be added to https connection's header. @@ -72,4 +72,4 @@ void initialize(Configuration configuration, String accountName) * @return Date to expire access token retrieved from AAD. */ Date getExpiryTime(); -} \ No newline at end of file +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java new file mode 100644 index 0000000000..667b7b2e07 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@InterfaceAudience.Public +package org.apache.hadoop.fs.azurebfs.extensions; +import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java index 7bae415daf..6e9f6350a1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee; /** * Provides tokens based on custom implementation, following the Adapter Design diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenIdentifier.java new file mode 100644 index 0000000000..390c2f4031 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenIdentifier.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.fs.azurebfs.security; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; + +/** + * Delegation token Identifier for ABFS delegation tokens. + */ +public class AbfsDelegationTokenIdentifier extends DelegationTokenIdentifier { + public static final Text TOKEN_KIND = new Text("ABFS delegation"); + + public AbfsDelegationTokenIdentifier(){ + super(TOKEN_KIND); + } + + public AbfsDelegationTokenIdentifier(Text kind) { + super(kind); + } + + public AbfsDelegationTokenIdentifier(Text kind, Text owner, Text renewer, + Text realUser) { + super(kind, owner, renewer, realUser); + } + + @Override + public Text getKind() { + return TOKEN_KIND; + } + +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java new file mode 100644 index 0000000000..eb47f768a4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.fs.azurebfs.security; + +import java.io.IOException; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.extensions.CustomDelegationTokenManager; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Class for delegation token Manager. + */ +public class AbfsDelegationTokenManager { + + private CustomDelegationTokenManager tokenManager; + private static final Logger LOG = + LoggerFactory.getLogger(AbfsDelegationTokenManager.class); + + public AbfsDelegationTokenManager(final Configuration conf) throws IOException { + + Preconditions.checkNotNull(conf, "conf"); + + Class customDelegationTokenMgrClass = + conf.getClass(ConfigurationKeys.FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE, null, + CustomDelegationTokenManager.class); + + if (customDelegationTokenMgrClass == null) { + throw new IllegalArgumentException( + "The value for \"fs.azure.delegation.token.provider.type\" is not defined."); + } + + CustomDelegationTokenManager customTokenMgr = (CustomDelegationTokenManager) ReflectionUtils + .newInstance(customDelegationTokenMgrClass, conf); + if (customTokenMgr == null) { + throw new IllegalArgumentException(String.format("Failed to initialize %s.", customDelegationTokenMgrClass)); + } + + customTokenMgr.initialize(conf); + + tokenManager = customTokenMgr; + } + + public Token getDelegationToken( + String renewer) throws IOException { + + Token token = tokenManager.getDelegationToken(renewer); + + token.setKind(AbfsDelegationTokenIdentifier.TOKEN_KIND); + return token; + } + + public long renewDelegationToken(Token token) + throws IOException { + + return tokenManager.renewDelegationToken(token); + } + + public void cancelDelegationToken(Token token) + throws IOException { + + tokenManager.cancelDelegationToken(token); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsTokenRenewer.java new file mode 100644 index 0000000000..ab51838f26 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsTokenRenewer.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.security; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenRenewer; + +/** + * Token Renewer for renewing ABFS delegation tokens with remote service. + */ +public class AbfsTokenRenewer extends TokenRenewer { + public static final Logger LOG = + LoggerFactory.getLogger(AbfsTokenRenewer.class); + + /** + * Checks if this particular object handles the Kind of token passed. + * + * @param kind the kind of the token + * @return true if it handles passed token kind false otherwise. + */ + @Override + public boolean handleKind(Text kind) { + return AbfsDelegationTokenIdentifier.TOKEN_KIND.equals(kind); + } + + /** + * Checks if passed token is managed. + * + * @param token the token being checked + * @return true if it is managed. + * @throws IOException thrown when evaluating if token is managed. + */ + @Override + public boolean isManaged(Token token) throws IOException { + return true; + } + + /** + * Renew the delegation token. + * + * @param token token to renew. + * @param conf configuration object. + * @return extended expiry time of the token. + * @throws IOException thrown when trying get current user. + * @throws InterruptedException thrown when thread is interrupted + */ + @Override + public long renew(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Renewing the delegation token"); + return getInstance(conf).renewDelegationToken(token); + } + + /** + * Cancel the delegation token. + * + * @param token token to cancel. + * @param conf configuration object. + * @throws IOException thrown when trying get current user. + * @throws InterruptedException thrown when thread is interrupted. + */ + @Override + public void cancel(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Cancelling the delegation token"); + getInstance(conf).cancelDelegationToken(token); + } + + private AbfsDelegationTokenManager getInstance(Configuration conf) + throws IOException { + return new AbfsDelegationTokenManager(conf); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/package-info.java new file mode 100644 index 0000000000..7c3e37ae6e --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.azurebfs.security; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index 7ec8216deb..9016918586 100644 --- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenIdentifier org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index f9c590aad8..d889534c73 100644 --- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +org.apache.hadoop.fs.azurebfs.security.AbfsTokenRenewer org.apache.hadoop.fs.azure.security.WasbTokenRenewer \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 9daac2ae51..88f77b0bca 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -56,6 +56,7 @@ public void testEnsureStatusWorksForRoot() throws Exception { @Test public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); + fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); touch(TEST_FILE); validateStatus(fs, TEST_FILE, false); } @@ -63,7 +64,6 @@ public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, final boolean isDir) throws IOException { FileStatus fileStatus = fs.getFileStatus(name); - fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); String errorInStatus = "error in " + fileStatus + " from " + fs; @@ -89,6 +89,7 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, @Test public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); + fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); fs.mkdirs(TEST_FOLDER); validateStatus(fs, TEST_FOLDER, true);