big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17953. S3A: Tests to lookup global or per-bucket configuration for encryption algorithm (#3525)

Browse Source 
Followup to S3-CSE work of HADOOP-13887

Contributed by Mehakmeet Singh
This commit is contained in:
Mehakmeet Singh 2021-10-19 15:28:27 +05:30 committed by GitHub
parent 5337bebcc5
commit cb8c98fbb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
View File

@ -178,7 +178,8 @@ public Boolean get() {
* *
* @param stats * @param stats
*/ */
protected abstract void verifyWrittenBytes(Statistics stats); protected abstract void verifyWrittenBytes(Statistics stats)
throws IOException;
/** /**
* Returns the filesystem uri. Should be set * Returns the filesystem uri. Should be set

14
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
View File

@ -22,6 +22,7 @@
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.PrintStream; import java.io.PrintStream;
import java.io.UncheckedIOException;
import java.net.URI; import java.net.URI;
import java.nio.file.AccessDeniedException; import java.nio.file.AccessDeniedException;
@ -41,7 +42,6 @@
import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Invoker;
import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.fs.s3a.S3ClientFactory; import org.apache.hadoop.fs.s3a.S3ClientFactory;
@ -69,6 +69,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders; import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders;
import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH;
@ -145,9 +146,14 @@ protected Configuration createConfiguration() {
// disable if assume role opts are off // disable if assume role opts are off
assumeSessionTestsEnabled(conf); assumeSessionTestsEnabled(conf);
disableFilesystemCaching(conf); disableFilesystemCaching(conf);
String s3EncryptionMethod = String s3EncryptionMethod;
conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM, try {
S3AEncryptionMethods.SSE_KMS.getMethod()); s3EncryptionMethod =
getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod();
} catch (IOException e) {
throw new UncheckedIOException("Failed to lookup encryption algorithm.",
e);
}
String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf); String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf);
removeBaseAndBucketOverrides(conf, removeBaseAndBucketOverrides(conf,
DELEGATION_TOKEN_BINDING, DELEGATION_TOKEN_BINDING,

16
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
View File

@ -13,6 +13,7 @@
*/ */
package org.apache.hadoop.fs.s3a.fileContext; package org.apache.hadoop.fs.s3a.fileContext;
import java.io.IOException;
import java.net.URI; import java.net.URI;
import com.amazonaws.services.s3.model.CryptoStorageMode; import com.amazonaws.services.s3.model.CryptoStorageMode;
@ -32,9 +33,10 @@
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
/** /**
@ -83,12 +85,14 @@ protected void verifyReadBytes(FileSystem.Statistics stats) {
* @param stats Filesystem statistics. * @param stats Filesystem statistics.
*/ */
@Override @Override
protected void verifyWrittenBytes(FileSystem.Statistics stats) { protected void verifyWrittenBytes(FileSystem.Statistics stats)
throws IOException {
//No extra bytes are written //No extra bytes are written
long expectedBlockSize = blockSize; long expectedBlockSize = blockSize;
if (conf.get(S3_ENCRYPTION_ALGORITHM, "") if (S3AEncryptionMethods.CSE_KMS.getMethod()
.equals(S3AEncryptionMethods.CSE_KMS.getMethod())) { .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf)
String keyId = conf.get(S3_ENCRYPTION_KEY, ""); .getMethod())) {
String keyId = getS3EncryptionKey(getTestBucketName(conf), conf);
// Adding padding length and KMS key generation bytes written. // Adding padding length and KMS key generation bytes written.
expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length + expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length +
KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;