From cb8c98fbb02847916e23ffaa60b0b54694d056d7 Mon Sep 17 00:00:00 2001 From: Mehakmeet Singh Date: Tue, 19 Oct 2021 15:28:27 +0530 Subject: [PATCH] HADOOP-17953. S3A: Tests to lookup global or per-bucket configuration for encryption algorithm (#3525) Followup to S3-CSE work of HADOOP-13887 Contributed by Mehakmeet Singh --- .../apache/hadoop/fs/FCStatisticsBaseTest.java | 3 ++- .../ITestSessionDelegationInFileystem.java | 14 ++++++++++---- .../ITestS3AFileContextStatistics.java | 16 ++++++++++------ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java index 2aa5407c05..dc12f44fc2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java @@ -178,7 +178,8 @@ public Boolean get() { * * @param stats */ - protected abstract void verifyWrittenBytes(Statistics stats); + protected abstract void verifyWrittenBytes(Statistics stats) + throws IOException; /** * Returns the filesystem uri. Should be set diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 6aed9e7e31..47fad29215 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.AccessDeniedException; @@ -41,7 +42,6 @@ import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; import org.apache.hadoop.fs.s3a.Invoker; -import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ClientFactory; @@ -69,6 +69,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH; @@ -145,9 +146,14 @@ protected Configuration createConfiguration() { // disable if assume role opts are off assumeSessionTestsEnabled(conf); disableFilesystemCaching(conf); - String s3EncryptionMethod = - conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM, - S3AEncryptionMethods.SSE_KMS.getMethod()); + String s3EncryptionMethod; + try { + s3EncryptionMethod = + getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to lookup encryption algorithm.", + e); + } String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf); removeBaseAndBucketOverrides(conf, DELEGATION_TOKEN_BINDING, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java index ca8ce15996..d28f4279f1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.fs.s3a.fileContext; +import java.io.IOException; import java.net.URI; import com.amazonaws.services.s3.model.CryptoStorageMode; @@ -32,9 +33,10 @@ import org.junit.Assert; import org.junit.Before; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; +import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; /** @@ -83,12 +85,14 @@ protected void verifyReadBytes(FileSystem.Statistics stats) { * @param stats Filesystem statistics. */ @Override - protected void verifyWrittenBytes(FileSystem.Statistics stats) { + protected void verifyWrittenBytes(FileSystem.Statistics stats) + throws IOException { //No extra bytes are written long expectedBlockSize = blockSize; - if (conf.get(S3_ENCRYPTION_ALGORITHM, "") - .equals(S3AEncryptionMethods.CSE_KMS.getMethod())) { - String keyId = conf.get(S3_ENCRYPTION_KEY, ""); + if (S3AEncryptionMethods.CSE_KMS.getMethod() + .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf) + .getMethod())) { + String keyId = getS3EncryptionKey(getTestBucketName(conf), conf); // Adding padding length and KMS key generation bytes written. expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length + KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;