From b2523d8100844338e073531c47666d744a101caf Mon Sep 17 00:00:00 2001 From: Da Zhou Date: Mon, 17 Dec 2018 11:04:40 +0000 Subject: [PATCH] HADOOP-15969. ABFS: getNamespaceEnabled can fail blocking user access thru ACLs. Contributed by Da Zhou. --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 19 ++-- .../azurebfs/AbstractAbfsIntegrationTest.java | 4 + .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 96 +++++++++++++++++++ .../constants/TestConfigurationKeys.java | 1 + 4 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 5f1692faa8..5d70b77de6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; @@ -181,12 +182,18 @@ private String[] authorityParts(URI uri) throws InvalidUriAuthorityException, In public boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException { if (!isNamespaceEnabledSet) { - LOG.debug("getFilesystemProperties for filesystem: {}", - client.getFileSystem()); - - final AbfsRestOperation op = client.getFilesystemProperties(); - isNamespaceEnabled = Boolean.parseBoolean( - op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_NAMESPACE_ENABLED)); + LOG.debug("Get root ACL status"); + try { + client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + AbfsHttpConstants.ROOT_PATH); + isNamespaceEnabled = true; + } catch (AbfsRestOperationException ex) { + // Get ACL status is a HEAD request, its response doesn't contain errorCode + // So can only rely on its status code to determine its account type. + if (HttpURLConnection.HTTP_BAD_REQUEST != ex.getStatusCode()) { + throw ex; + } + isNamespaceEnabled = false; + } isNamespaceEnabledSet = true; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 382cd7fbd6..cb9549d81d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -262,6 +262,10 @@ public AuthType getAuthType() { return this.authType; } + public String getAbfsScheme() { + return this.abfsScheme; + } + protected boolean isIPAddress() { return isIPAddress; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java new file mode 100644 index 0000000000..b9a2315669 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import java.io.IOException; +import java.util.UUID; + +import org.junit.Assume; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.services.AuthType; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Test getIsNamespaceEnabled call. + */ +public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest { + + private boolean isUsingXNSAccount; + public ITestGetNameSpaceEnabled() throws Exception { + isUsingXNSAccount = getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); + } + + @Test + public void testXNSAccount() throws IOException { + Assume.assumeTrue("Skip this test because the account being used for test is a non XNS account", + isUsingXNSAccount); + assertTrue("Expecting getIsNamespaceEnabled() return true", + getFileSystem().getIsNamespaceEnabled()); + } + + @Test + public void testNonXNSAccount() throws IOException { + Assume.assumeFalse("Skip this test because the account being used for test is a XNS account", + isUsingXNSAccount); + assertFalse("Expecting getIsNamespaceEnabled() return false", + getFileSystem().getIsNamespaceEnabled()); + } + + @Test + public void testFailedRequestWhenFSNotExist() throws Exception { + AbfsConfiguration config = this.getConfiguration(); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); + String testUri = this.getTestUrl(); + String nonExistingFsUrl = getAbfsScheme() + "://" + UUID.randomUUID() + + testUri.substring(testUri.indexOf("@")); + AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl); + + intercept(AbfsRestOperationException.class, + "\"The specified filesystem does not exist.\", 404", + ()-> { + fs.getIsNamespaceEnabled(); + }); + } + + @Test + public void testFailedRequestWhenCredentialsNotCorrect() throws Exception { + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + Configuration config = this.getRawConfiguration(); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); + String accountName = this.getAccountName(); + String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName; + // Provide a wrong sharedKey + String secret = config.get(configkKey); + secret = (char) (secret.charAt(0) + 1) + secret.substring(1); + config.set(configkKey, secret); + + AzureBlobFileSystem fs = this.getFileSystem(config); + intercept(AbfsRestOperationException.class, + "\"Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\", 403", + ()-> { + fs.getIsNamespaceEnabled(); + }); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java index 5565a4920e..fbd13fee86 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java @@ -26,6 +26,7 @@ public final class TestConfigurationKeys { public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name"; public static final String FS_AZURE_ACCOUNT_KEY = "fs.azure.account.key"; public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs"; + public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id"; public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET = "fs.azure.account.oauth2.contributor.client.secret";