big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17953. S3A: Tests to lookup global or per-bucket configuration for encryption algorithm (#3525)

Browse Source 
Followup to S3-CSE work of HADOOP-13887

Contributed by Mehakmeet Singh
This commit is contained in:
Mehakmeet Singh 2021-10-19 15:28:27 +05:30 committed by GitHub
parent 5337bebcc5
commit cb8c98fbb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
View File

@ -178,7 +178,8 @@ public Boolean get() {
*
* @param stats
*/
protected abstract void verifyWrittenBytes(Statistics stats);
protected abstract void verifyWrittenBytes(Statistics stats)
throws IOException;
/**
* Returns the filesystem uri. Should be set

14
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
View File

@ -22,6 +22,7 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
@ -41,7 +42,6 @@
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.fs.s3a.S3ClientFactory;
@ -69,6 +69,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders;
import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH;
@ -145,9 +146,14 @@ protected Configuration createConfiguration() {
// disable if assume role opts are off
assumeSessionTestsEnabled(conf);
disableFilesystemCaching(conf);
String s3EncryptionMethod =
conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM,
S3AEncryptionMethods.SSE_KMS.getMethod());
String s3EncryptionMethod;
try {
s3EncryptionMethod =
getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod();
} catch (IOException e) {
throw new UncheckedIOException("Failed to lookup encryption algorithm.",
e);
}
String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf);
removeBaseAndBucketOverrides(conf,
DELEGATION_TOKEN_BINDING,

16
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
View File

@ -13,6 +13,7 @@
*/
package org.apache.hadoop.fs.s3a.fileContext;
import java.io.IOException;
import java.net.URI;
import com.amazonaws.services.s3.model.CryptoStorageMode;
@ -32,9 +33,10 @@
import org.junit.Assert;
import org.junit.Before;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
/**
@ -83,12 +85,14 @@ protected void verifyReadBytes(FileSystem.Statistics stats) {
* @param stats Filesystem statistics.
*/
@Override
protected void verifyWrittenBytes(FileSystem.Statistics stats) {
protected void verifyWrittenBytes(FileSystem.Statistics stats)
throws IOException {
//No extra bytes are written
long expectedBlockSize = blockSize;
if (conf.get(S3_ENCRYPTION_ALGORITHM, "")
.equals(S3AEncryptionMethods.CSE_KMS.getMethod())) {
String keyId = conf.get(S3_ENCRYPTION_KEY, "");
if (S3AEncryptionMethods.CSE_KMS.getMethod()
.equals(getEncryptionAlgorithm(getTestBucketName(conf), conf)
.getMethod())) {
String keyId = getS3EncryptionKey(getTestBucketName(conf), conf);
// Adding padding length and KMS key generation bytes written.
expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length +
KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;