big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14802. Add support for using container saskeys for all accesses.

Browse Source 
Contributed by Sivaguru Sankaridurg
This commit is contained in:
Steve Loughran 2017-08-29 14:58:43 +01:00
parent 1f6dc4ee9a
commit 021974f4cb
No known key found for this signature in database
GPG Key ID: 950CC3E032B79CA2
5 changed files with 83 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1380,7 +1380,15 @@
This flag is relevant only when fs.azure.authorization is enabled.
</description>
</property>
<property>
<name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
<value>true</value>
<description>
Use container saskey for access to all blobs within the container.
Blob-specific saskeys are not used when this setting is enabled.
This setting provides better performance compared to blob-specific saskeys.
</description>
</property>
<property>
<name>io.seqfile.compress.blocksize</name>
<value>1000000</value>

1
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -123,6 +123,7 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("fs.azure.secure.mode");
xmlPropsToSkipCompare.add("fs.azure.authorization");
xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable");
xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
// Deprecated properties. These should eventually be removed from the

29
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
View File

@ -71,6 +71,13 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
private String storageAccount;
private RetryPolicyFactory retryPolicy;
private int timeoutIntervalInMs;
private boolean useContainerSasKeyForAllAccess;
/**
* Configuration key to specify if containerSasKey should be used for all accesses
*/
public static final String KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS =
"fs.azure.saskey.usecontainersaskeyforallaccess";
public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
Configuration conf) throws SecureModeException {
@ -88,6 +95,7 @@ public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
}
this.sasKeyGenerator = remoteSasKeyGenerator;
}
this.useContainerSasKeyForAllAccess = conf.getBoolean(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, true);
}
@Override
@ -145,7 +153,9 @@ public CloudBlobContainerWrapper getContainerReference(String name)
if (timeoutIntervalInMs > 0) {
container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
}
return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
return (useContainerSasKeyForAllAccess)
? new SASCloudBlobContainerWrapperImpl(storageAccount, container, null)
: new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for container : " + name
@ -226,12 +236,12 @@ public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
public CloudBlobWrapper getBlockBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(
storageAccount, getName(), relativePath));
CloudBlockBlob blob = (sasKeyGenerator!=null)
? new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
: container.getBlockBlobReference(relativePath);
blob.getServiceClient().setDefaultRequestOptions(
container.getServiceClient().getDefaultRequestOptions());
return new SASCloudBlockBlobWrapperImpl(
blob);
return new SASCloudBlockBlobWrapperImpl(blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath
@ -245,12 +255,13 @@ public CloudBlobWrapper getBlockBlobReference(String relativePath)
public CloudBlobWrapper getPageBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(
storageAccount, getName(), relativePath));
CloudPageBlob blob = (sasKeyGenerator!=null)
? new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
: container.getPageBlobReference(relativePath);
blob.getServiceClient().setDefaultRequestOptions(
container.getServiceClient().getDefaultRequestOptions());
return new SASCloudPageBlobWrapperImpl(
blob);
return new SASCloudPageBlobWrapperImpl(blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath

9
hadoop-tools/hadoop-azure/src/site/markdown/index.md
View File

@ -476,6 +476,15 @@ The maximum number of entries that that cache can hold can be customized using t
</property>
```
Use container saskey for access to all blobs within the container.
Blob-specific saskeys are not used when this setting is enabled.
This setting provides better performance compared to blob-specific saskeys.
```
<property>
<name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
<value>true</value>
</property>
```
## Testing the hadoop-azure Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests

44
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azure;
import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS;
/**
* Test class to hold all WASB authorization tests that use blob-specific keys
* to access storage.
*/
public class TestNativeAzureFSAuthWithBlobSpecificKeys
extends TestNativeAzureFileSystemAuthorizationWithOwner {
@Override
public Configuration getConfiguration() {
Configuration conf = super.getConfiguration();
conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false");
return conf;
}
@Override
protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
Configuration conf = getConfiguration();
return AzureBlobStorageTestAccount.create(conf);
}
}