From 021974f4cb1eb8cf530dabc95039fed46e66a43d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 29 Aug 2017 14:58:43 +0100 Subject: [PATCH] HADOOP-14802. Add support for using container saskeys for all accesses. Contributed by Sivaguru Sankaridurg --- .../src/main/resources/core-default.xml | 10 ++++- .../conf/TestCommonConfigurationFields.java | 1 + .../fs/azure/SecureStorageInterfaceImpl.java | 29 ++++++++---- .../hadoop-azure/src/site/markdown/index.md | 9 ++++ ...NativeAzureFSAuthWithBlobSpecificKeys.java | 44 +++++++++++++++++++ 5 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 7c4b0f1f98..cb061aaddc 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1380,7 +1380,15 @@ This flag is relevant only when fs.azure.authorization is enabled. - + + fs.azure.saskey.usecontainersaskeyforallaccess + true + + Use container saskey for access to all blobs within the container. + Blob-specific saskeys are not used when this setting is enabled. + This setting provides better performance compared to blob-specific saskeys. + + io.seqfile.compress.blocksize 1000000 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index d0e0a351b5..33248864f2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -123,6 +123,7 @@ public void initializeMemberVariables() { xmlPropsToSkipCompare.add("fs.azure.secure.mode"); xmlPropsToSkipCompare.add("fs.azure.authorization"); xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable"); + xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess"); xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix"); // Deprecated properties. These should eventually be removed from the diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java index 3d33453e93..5dbb6bc8c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java @@ -71,6 +71,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface { private String storageAccount; private RetryPolicyFactory retryPolicy; private int timeoutIntervalInMs; + private boolean useContainerSasKeyForAllAccess; + + /** + * Configuration key to specify if containerSasKey should be used for all accesses + */ + public static final String KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS = + "fs.azure.saskey.usecontainersaskeyforallaccess"; public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, Configuration conf) throws SecureModeException { @@ -88,6 +95,7 @@ public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, } this.sasKeyGenerator = remoteSasKeyGenerator; } + this.useContainerSasKeyForAllAccess = conf.getBoolean(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, true); } @Override @@ -145,7 +153,9 @@ public CloudBlobContainerWrapper getContainerReference(String name) if (timeoutIntervalInMs > 0) { container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs); } - return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator); + return (useContainerSasKeyForAllAccess) + ? new SASCloudBlobContainerWrapperImpl(storageAccount, container, null) + : new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for container : " + name @@ -226,12 +236,12 @@ public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath) public CloudBlobWrapper getBlockBlobReference(String relativePath) throws URISyntaxException, StorageException { try { - CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri( - storageAccount, getName(), relativePath)); + CloudBlockBlob blob = (sasKeyGenerator!=null) + ? new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath)) + : container.getBlockBlobReference(relativePath); blob.getServiceClient().setDefaultRequestOptions( container.getServiceClient().getDefaultRequestOptions()); - return new SASCloudBlockBlobWrapperImpl( - blob); + return new SASCloudBlockBlobWrapperImpl(blob); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for relativePath : " + relativePath @@ -245,12 +255,13 @@ public CloudBlobWrapper getBlockBlobReference(String relativePath) public CloudBlobWrapper getPageBlobReference(String relativePath) throws URISyntaxException, StorageException { try { - CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri( - storageAccount, getName(), relativePath)); + CloudPageBlob blob = (sasKeyGenerator!=null) + ? new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath)) + : container.getPageBlobReference(relativePath); + blob.getServiceClient().setDefaultRequestOptions( container.getServiceClient().getDefaultRequestOptions()); - return new SASCloudPageBlobWrapperImpl( - blob); + return new SASCloudPageBlobWrapperImpl(blob); } catch (SASKeyGenerationException sasEx) { String errorMsg = "Encountered SASKeyGeneration exception while " + "generating SAS Key for relativePath : " + relativePath diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 79cb0eafb1..758650daa9 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -476,6 +476,15 @@ The maximum number of entries that that cache can hold can be customized using t ``` + Use container saskey for access to all blobs within the container. + Blob-specific saskeys are not used when this setting is enabled. + This setting provides better performance compared to blob-specific saskeys. + ``` + + fs.azure.saskey.usecontainersaskeyforallaccess + true + +``` ## Testing the hadoop-azure Module The hadoop-azure module includes a full suite of unit tests. Most of the tests diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java new file mode 100644 index 0000000000..6149154a27 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS; + +/** + * Test class to hold all WASB authorization tests that use blob-specific keys + * to access storage. + */ +public class TestNativeAzureFSAuthWithBlobSpecificKeys + extends TestNativeAzureFileSystemAuthorizationWithOwner { + + @Override + public Configuration getConfiguration() { + Configuration conf = super.getConfiguration(); + conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false"); + return conf; + } + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + Configuration conf = getConfiguration(); + return AzureBlobStorageTestAccount.create(conf); + } +}