From df2166a6435b4b62011a363ea24dfcc9bc44181c Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 18 Sep 2018 12:20:52 +0100 Subject: [PATCH] HADOOP-15719. Fail-fast when using OAuth over http. Contributed by Da Zhou. --- .../fs/azurebfs/AzureBlobFileSystem.java | 13 ++--- .../fs/azurebfs/AzureBlobFileSystemStore.java | 7 +++ ...onfigurationPropertyNotFoundException.java | 2 +- .../services/TestOauthFailOverHttp.java | 55 +++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestOauthFailOverHttp.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 2e8de78288..5605e362ea 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -101,12 +101,11 @@ public void initialize(URI uri, Configuration configuration) this.userGroupInformation = UserGroupInformation.getCurrentUser(); this.user = userGroupInformation.getUserName(); this.abfsStore = new AzureBlobFileSystemStore(uri, this.isSecure(), configuration, userGroupInformation); - - LOG.debug("Initializing NativeAzureFileSystem for {}", uri); + final AbfsConfiguration abfsConfiguration = abfsStore.getAbfsConfiguration(); this.setWorkingDirectory(this.getHomeDirectory()); - if (abfsStore.getAbfsConfiguration().getCreateRemoteFileSystemDuringInitialization()) { + if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { if (!this.fileSystemExists()) { try { this.createFileSystem(); @@ -116,7 +115,7 @@ public void initialize(URI uri, Configuration configuration) } } - if (!abfsStore.getAbfsConfiguration().getSkipUserGroupMetadataDuringInitialization()) { + if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) { this.primaryUserGroup = userGroupInformation.getPrimaryGroupName(); } else { //Provide a default group name @@ -124,15 +123,15 @@ public void initialize(URI uri, Configuration configuration) } if (UserGroupInformation.isSecurityEnabled()) { - this.delegationTokenEnabled = abfsStore.getAbfsConfiguration().isDelegationTokenManagerEnabled(); + this.delegationTokenEnabled = abfsConfiguration.isDelegationTokenManagerEnabled(); if (this.delegationTokenEnabled) { LOG.debug("Initializing DelegationTokenManager for {}", uri); - this.delegationTokenManager = abfsStore.getAbfsConfiguration().getDelegationTokenManager(); + this.delegationTokenManager = abfsConfiguration.getDelegationTokenManager(); } } - AbfsClientThrottlingIntercept.initializeSingleton(abfsStore.getAbfsConfiguration().isAutoThrottlingEnabled()); + AbfsClientThrottlingIntercept.initializeSingleton(abfsConfiguration.isAutoThrottlingEnabled()); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index d16cf3622b..cf7387b6e6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -88,6 +88,7 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; import static org.apache.hadoop.util.Time.now; /** @@ -130,6 +131,12 @@ public AzureBlobFileSystemStore(URI uri, boolean isSecure, Configuration configu this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList( abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA))); + if (AuthType.OAuth == abfsConfiguration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey) + && !FileSystemUriSchemes.ABFS_SECURE_SCHEME.equals(uri.getScheme())) { + throw new IllegalArgumentException( + String.format("Incorrect URI %s, URI scheme must be abfss when authenticating using Oauth.", uri)); + } + initializeClient(uri, fileSystemName, accountName, isSecure); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java index bf3b2f34d6..43a71ab43c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/ConfigurationPropertyNotFoundException.java @@ -27,6 +27,6 @@ @InterfaceStability.Evolving public class ConfigurationPropertyNotFoundException extends AzureBlobFileSystemException { public ConfigurationPropertyNotFoundException(String property) { - super("Configuration property " + property + "not found."); + super("Configuration property " + property + " not found."); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestOauthFailOverHttp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestOauthFailOverHttp.java new file mode 100644 index 0000000000..de07c4b2b9 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestOauthFailOverHttp.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.services; + +import java.net.URI; + +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Test Oauth fail fast when uri scheme is incorrect. + */ +public class TestOauthFailOverHttp { + + @Test + public void testOauthFailWithSchemeAbfs() throws Exception { + Configuration conf = new Configuration(); + final String account = "fakeaccount.dfs.core.windows.net"; + conf.set(FS_AZURE_ABFS_ACCOUNT_NAME, account); + conf.setEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.OAuth); + URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SCHEME, + "fakecontainer@" + account, + null, + null, + null); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + // IllegalArgumentException is expected + // when authenticating using Oauth and scheme is not abfss + intercept(IllegalArgumentException.class, "Incorrect URI", + () -> FileSystem.get(conf)); + } +}