diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 224cdeffe7..75b09a4ed7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -61,7 +61,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class AWSCredentialProviderList implements AWSCredentialsProvider, +public final class AWSCredentialProviderList implements AWSCredentialsProvider, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index 0cb9fd4e57..564c03bf73 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -34,9 +34,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Stable +@Deprecated public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 306fd8e1c6..6d20e3b085 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -140,6 +140,7 @@ private Constants() { public static final String ASSUMED_ROLE_POLICY = "fs.s3a.assumed.role.policy"; + @SuppressWarnings("deprecation") public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT = SimpleAWSCredentialsProvider.NAME; @@ -732,6 +733,7 @@ private Constants() { @InterfaceAudience.Private @InterfaceStability.Unstable + @SuppressWarnings("deprecation") public static final Class DEFAULT_S3_CLIENT_FACTORY_IMPL = DefaultS3ClientFactory.class; @@ -1204,6 +1206,11 @@ private Constants() { */ public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M + /** + * Prefix of auth classes in AWS SDK V1. + */ + public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth"; + /** * Maximum number of range reads a single input stream can have * active (downloading, or queued) to the central FileSystem diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index f12a7e089f..bab4ec896f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -71,6 +71,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class DefaultS3ClientFactory extends Configured implements S3ClientFactory { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index aa6f69c2a2..9d655a8cdd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -54,6 +54,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; @@ -70,6 +72,8 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; @@ -126,6 +130,7 @@ import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; import org.apache.hadoop.fs.statistics.DurationTracker; @@ -853,6 +858,7 @@ public Listing getListing() { * @param dtEnabled are delegation tokens enabled? * @throws IOException failure. */ + @SuppressWarnings("deprecation") private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { Configuration conf = getConf(); credentials = null; @@ -865,6 +871,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); + V2Migration.v1DelegationTokenCredentialProvidersUsed(); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), @@ -1192,7 +1199,7 @@ public int getDefaultPort() { * This is for internal use within the S3A code itself. * @return AmazonS3Client */ - AmazonS3 getAmazonS3Client() { + private AmazonS3 getAmazonS3Client() { return s3; } @@ -1206,6 +1213,7 @@ AmazonS3 getAmazonS3Client() { @VisibleForTesting public AmazonS3 getAmazonS3ClientForTesting(String reason) { LOG.warn("Access to S3A client requested, reason {}", reason); + V2Migration.v1S3ClientRequested(); return s3; } @@ -1586,6 +1594,25 @@ public CompletableFuture submit(final CallableRaisingIOE operation) { } } + /** + * Callbacks for WriteOperationHelper. + */ + private final class WriteOperationHelperCallbacksImpl + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return s3.selectObjectContent(request); + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return s3.completeMultipartUpload(request); + } + } + + /** * Create the read context for reading from the referenced file, * using FS state as well as the status. @@ -1806,7 +1833,8 @@ public WriteOperationHelper createWriteOperationHelper(AuditSpan auditSpan) { getConf(), statisticsContext, getAuditSpanSource(), - auditSpan); + auditSpan, + new WriteOperationHelperCallbacksImpl()); } /** @@ -2287,6 +2315,7 @@ public int getMaxKeys() { @Retries.RetryTranslated @InterfaceStability.Evolving public ObjectMetadata getObjectMetadata(Path path) throws IOException { + V2Migration.v1GetObjectMetadataCalled(); return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () -> getObjectMetadata(makeQualified(path), null, invoker, "getObjectMetadata")); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index ace3674408..72a1784dd0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.impl.NetworkBinding; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.security.ProviderUtils; @@ -550,6 +551,7 @@ public static long dateToLong(final Date date) { /** * The standard AWS provider list for AWS connections. */ + @SuppressWarnings("deprecation") public static final List> STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( Arrays.asList( @@ -636,6 +638,10 @@ public static AWSCredentialProviderList buildAWSProviderList( AWSCredentialProviderList providers = new AWSCredentialProviderList(); for (Class aClass : awsClasses) { + if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { + V2Migration.v1ProviderReferenced(aClass.getName()); + } + if (forbidden.contains(aClass)) { throw new IOException(E_FORBIDDEN_AWS_PROVIDER + " in option " + key + ": " + aClass); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 2216599679..9010f34dc2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -44,9 +44,12 @@ * implementing only the deprecated method will work. * See https://github.com/apache/hbase-filesystem * + * @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of + * upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.LimitedPrivate("HBoss") @InterfaceStability.Evolving +@Deprecated public interface S3ClientFactory { /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index 5eba675cb8..6579a2bc3e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -39,6 +39,6 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -public final class SharedInstanceCredentialProvider extends - IAMInstanceCredentialsProvider { +@SuppressWarnings("deprecation") +public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index ee689e58b7..657bd437b5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -41,9 +41,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable +@Deprecated public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index 9c71ab4586..db3d0bb132 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -43,11 +43,14 @@ * * This credential provider must not fail in creation because that will * break a chain of credential providers. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable -public class TemporaryAWSCredentialsProvider extends - AbstractSessionCredentialsProvider { +@Deprecated +public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 12430fce0d..060a3c3b96 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -144,6 +144,11 @@ public class WriteOperationHelper implements WriteOperations { */ private final RequestFactory requestFactory; + /** + * WriteOperationHelper callbacks. + */ + private final WriteOperationHelperCallbacks writeOperationHelperCallbacks; + /** * Constructor. * @param owner owner FS creating the helper @@ -151,13 +156,14 @@ public class WriteOperationHelper implements WriteOperations { * @param statisticsContext statistics context * @param auditSpanSource source of spans * @param auditSpan span to activate - * + * @param writeOperationHelperCallbacks callbacks used by writeOperationHelper */ protected WriteOperationHelper(S3AFileSystem owner, Configuration conf, S3AStatisticsContext statisticsContext, final AuditSpanSource auditSpanSource, - final AuditSpan auditSpan) { + final AuditSpan auditSpan, + final WriteOperationHelperCallbacks writeOperationHelperCallbacks) { this.owner = owner; this.invoker = new Invoker(new S3ARetryPolicy(conf), this::operationRetried); @@ -168,6 +174,7 @@ protected WriteOperationHelper(S3AFileSystem owner, this.auditSpanSource = auditSpanSource; this.auditSpan = checkNotNull(auditSpan); this.requestFactory = owner.getRequestFactory(); + this.writeOperationHelperCallbacks = writeOperationHelperCallbacks; } /** @@ -359,8 +366,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( final CompleteMultipartUploadRequest request = getRequestFactory().newCompleteMultipartUploadRequest( destKey, uploadId, partETags); - return owner.getAmazonS3Client().completeMultipartUpload( - request); + return writeOperationHelperCallbacks.completeMultipartUpload(request); }); owner.finishedWrite(destKey, length, uploadResult.getETag(), uploadResult.getVersionId(), @@ -716,7 +722,7 @@ public SelectObjectContentResult select( try (DurationInfo ignored = new DurationInfo(LOG, "S3 Select operation")) { try { - return owner.getAmazonS3Client().selectObjectContent(request); + return writeOperationHelperCallbacks.selectObjectContent(request); } catch (AmazonS3Exception e) { LOG.error("Failure of S3 Select request against {}", source); @@ -758,4 +764,25 @@ public RequestFactory getRequestFactory() { return requestFactory; } + /*** + * Callbacks for writeOperationHelper. + */ + public interface WriteOperationHelperCallbacks { + + /** + * Initiates a select request. + * @param request selectObjectContent request + * @return selectObjectContentResult + */ + SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request); + + /** + * Initiates a complete multi-part upload request. + * @param request Complete multi-part upload request + * @return completeMultipartUploadResult + */ + CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request); + + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java index 1f714b0555..1815285738 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java @@ -28,7 +28,11 @@ /** * Base class for AWS credential providers which * take a URI and config in their constructor. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ +@Deprecated public abstract class AbstractAWSCredentialProvider implements AWSCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java index 816f3e47da..cab3aa9905 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java @@ -35,8 +35,12 @@ /** * Base class for session credential support. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private +@Deprecated public abstract class AbstractSessionCredentialsProvider extends AbstractAWSCredentialProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index c467702575..fe816ecfcd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -60,9 +60,13 @@ * unless overridden, creating a session name from the current user. * * Classname is used in configuration files; do not move. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index 1bb30ed5c0..ca9c518d30 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -40,9 +40,13 @@ * as a non-recoverable failure. *

* It is implicitly public; marked evolving as we can change its semantics. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class IAMInstanceCredentialsProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index b0a1272256..cde4cc94b7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -40,6 +40,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index cda769a789..e162428787 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -70,6 +71,8 @@ public void initCustomSigners() { return; } + V2Migration.v1CustomSignerUsed(); + for (String customSigner : customSigners) { String[] parts = customSigner.split(":"); if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 364966e308..f894479646 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -120,6 +120,12 @@ private InternalConstants() { public static final String THROTTLE_LOG_NAME = "org.apache.hadoop.fs.s3a.throttled"; + /** + * Name of the log for events related to the SDK V2 upgrade. + */ + public static final String SDK_V2_UPGRADE_LOG_NAME = + "org.apache.hadoop.fs.s3a.SDKV2Upgrade"; + /** Directory marker attribute: see HADOOP-16613. Value: {@value}. */ public static final String X_DIRECTORY = "application/x-directory"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java new file mode 100644 index 0000000000..3aa8ad270e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.store.LogExactlyOnce; + +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; + +/** + * This class provides utility methods required for migrating S3A to AWS Java SDK V2. + * For more information on the upgrade, see HADOOP-18073. + */ +public final class V2Migration { + + private V2Migration() { } + + public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); + + private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + /** + * Warns on an AWS V1 credential provider being referenced directly. + * @param name name of the credential provider + */ + public static void v1ProviderReferenced(String name) { + WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( + "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " + + "providers will be removed once S3A is upgraded to SDK V2", name); + } + + /** + * Warns on the v1 s3 client being requested. + */ + public static void v1S3ClientRequested() { + WARN_ON_GET_S3_CLIENT.warn( + "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); + } + + /** + * Warns when v1 credential providers are used with delegation tokens. + */ + public static void v1DelegationTokenCredentialProvidersUsed() { + WARN_ON_DELEGATION_TOKENS.warn( + "The credential provider interface has changed in AWS SDK V2, custom credential " + + "providers used in delegation tokens binding classes will need to be updated once " + + "S3A is upgraded to SDK V2"); + } + + /** + * Warns on use of custom signers. + */ + public static void v1CustomSignerUsed() { + WARN_OF_CUSTOM_SIGNER.warn( + "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " + + "once S3A is upgraded to SDK V2"); + } + + /** + * Warns on use of getObjectMetadata. + */ + public static void v1GetObjectMetadataCalled() { + WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response " + + "will be changed as part of upgrading S3A to AWS SDK V2"); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md new file mode 100644 index 0000000000..e649a8d76d --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -0,0 +1,76 @@ + + +# Upgrading S3A to AWS SDK V2 + +This document explains the upcoming work for upgrading S3A to AWS SDK V2. +This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). + +## Why the upgrade? + +- Moving to SDK V2 will provide performance benefits. +For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) +is built using java bindings of the AWS Common Runtime S3 +client (https://github.com/awslabs/aws-crt-java) (CRT). +CRT is a set of packages written in C, designed for maximising performance when interacting with AWS +services such as S3. +- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) +which S3A will benefit from are not available in SDKV1. + +## What's changing? + +The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from +[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. +A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). + +The major changes and how this affects S3A are listed below. + +### Package Change + +Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes +were under `com.amazonaws`. + +### Credential Providers + +- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java) +has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java). +- Credential provider class changes: the package and class names of credential providers have +changed. + +The change in interface will mean that custom credential providers will need to be updated to now +implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. + +Due to change in class names, references to SDK V1 credential providers +in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers. + +### Delegation Tokens + +Custom credential providers used in delegation token binding classes will also need to be updated. + +### AmazonS3 replaced by S3Client + +The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. + +For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 +`S3Client`. + +The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. + +### Signers + +Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java) +has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java). + +The change in signers will mean the custom signers will need to be updated to implement the new +interface. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 049087f0af..6a7d061e54 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -47,6 +47,7 @@ full details. * [Auditing](./auditing.html). * [Auditing Architecture](./auditing_architecture.html). * [Testing](./testing.html) +* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html) ## Overview @@ -234,6 +235,9 @@ needs the credentials needed to interact with buckets. The client supports multiple authentication mechanisms and can be configured as to which mechanisms to use, and their order of use. Custom implementations of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. +However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be +updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. +For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). *Important*: The S3A connector no longer supports username and secrets in URLs of the form `s3a://key:secret@bucket/`. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 903310a94b..ef9f7f123e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -21,12 +21,13 @@ Common problems working with S3 are: 1. [Classpath setup](#classpath) -1. [Authentication](#authentication) -1. [Access Denial](#access_denied) -1. [Connectivity Problems](#connectivity) -1. [File System Semantics](#semantics) -1. [Encryption](#encryption) -1. [Other Errors](#other) +2. [Authentication](#authentication) +3. [Access Denial](#access_denied) +4. [Connectivity Problems](#connectivity) +5. [File System Semantics](#semantics) +6. [Encryption](#encryption) +7. [Other Errors](#other) +8. [SDK Upgrade Warnings](#upgrade_warnings) This document also includes some [best pactises](#best) to aid troubleshooting. @@ -1989,3 +1990,51 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request: When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it completely by setting it to `0`. + +## SDK Upgrade Warnings + +S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2). +For more information on the upgrade and what's changing, see +[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). + +S3A logs the following warnings for things that will be changing in the upgrade. To disable these +logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties. + +### `Directly referencing AWS SDK V1 credential provider` + +This will be logged when an AWS credential provider is referenced directly in +`fs.s3a.aws.credentials.provider`. +For example, `com.amazonaws.auth.AWSSessionCredentialsProvider` + +To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`. +Instead, use S3A's credential providers. + +### `getAmazonS3ClientForTesting() will be removed` + +This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2, +the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to +`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed. + +### +### `Custom credential providers used in delegation tokens binding classes will need to be updated` + +This will be logged when delegation tokens are used. +Delegation tokens allow the use of custom binding classes which can implement custom credential +providers. +These credential providers will currently be implementing +`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement +`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. + +### +### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated` + +This will be logged when a custom signer is used. +Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be +updated to implement `software.amazon.awssdk.core.signer.Signer`. + +### +### `getObjectMetadata() called. This operation and it's response will be changed` + +This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to +`headObject()` and will return a response of the type `HeadObjectResponse`. + diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java index a80a24881f..a46303f339 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java @@ -62,6 +62,7 @@ public void setup() throws Exception { s3 = fs.getAmazonS3ClientForTesting("mocking"); } + @SuppressWarnings("deprecation") public Configuration createConfiguration() { Configuration conf = new Configuration(); conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index f1805b8408..c13c3f48b8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -157,6 +157,7 @@ public void testBadCredentials() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAnonymousProvider() throws Exception { Configuration conf = new Configuration(); conf.set(AWS_CREDENTIALS_PROVIDER, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 61d12747c0..26d00bc7d3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -414,6 +414,7 @@ public void testRequestTimeout() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testCloseIdempotent() throws Throwable { conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 40217484f2..add6502d7d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -165,6 +165,7 @@ private void createMarsNorth2Client(Configuration conf) throws Exception { * @throws URISyntaxException parse problems. * @throws IOException IO problems */ + @SuppressWarnings("deprecation") private AmazonS3 createS3Client(Configuration conf, String endpoint, String expectedRegion) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 35525eb1fc..0778662542 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -71,6 +71,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class); + @SuppressWarnings("deprecation") private static final String TEMPORARY_AWS_CREDENTIALS = TemporaryAWSCredentialsProvider.NAME; @@ -175,6 +176,7 @@ protected String getStsRegion(final Configuration conf) { } @Test + @SuppressWarnings("deprecation") public void testTemporaryCredentialValidation() throws Throwable { Configuration conf = new Configuration(); conf.set(ACCESS_KEY, "accesskey"); @@ -358,6 +360,7 @@ public void testSessionCredentialsEndpointNoRegion() throws Throwable { * @return the caught exception. * @throws Exception any unexpected exception. */ + @SuppressWarnings("deprecation") public E expectedSessionRequestFailure( final Class clazz, final String endpoint, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index edc2db8546..42fc630924 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -47,6 +47,7 @@ import org.apache.hadoop.fs.s3a.impl.StubContextAccessor; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.util.Progressable; @@ -176,7 +177,8 @@ public void initialize(URI name, Configuration originalConf) conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index bd121ba272..3240309aef 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -31,6 +31,7 @@ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3} * interface suitable for unit testing. */ +@SuppressWarnings("deprecation") public class MockS3ClientFactory implements S3ClientFactory { @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 260c5fc504..f3524c6684 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -607,6 +607,7 @@ public static void unsetHadoopCredentialProviders(final Configuration conf) { * @return a set of credentials * @throws IOException on a failure */ + @SuppressWarnings("deprecation") public static AWSCredentialsProvider buildAwsCredentialsProvider( final Configuration conf) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 08d5e0db0d..2c52bdc760 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -95,6 +95,7 @@ public void testProviderFailureError() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testInstantiationChain() throws Throwable { Configuration conf = new Configuration(false); conf.set(AWS_CREDENTIALS_PROVIDER, @@ -114,6 +115,7 @@ public void testInstantiationChain() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -138,6 +140,7 @@ public void testDefaultChainNoURI() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); List> expectedClasses = @@ -156,6 +159,7 @@ public void testConfiguredChain() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -368,6 +372,7 @@ private static void assertCredentialProviders( * @see S3ATestUtils#authenticationContains(Configuration, String). */ @Test + @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); assertFalse("found AssumedRoleCredentialProvider", diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index d38b5c93a6..ffa2c81e58 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.s3a.commit.PutTracker; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.util.Progressable; import org.junit.Before; @@ -102,7 +103,8 @@ public void testWriteOperationHelperPartLimits() throws Throwable { conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); ByteArrayInputStream inputStream = new ByteArrayInputStream( "a".getBytes()); // first one works diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 51eac7e8cc..9fb09b4ced 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -140,6 +140,7 @@ private E expectFileSystemCreateFailure( } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProvider() throws IOException { describe("Create the credential provider"); @@ -153,6 +154,7 @@ public void testCreateCredentialProvider() throws IOException { } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProviderNoURI() throws IOException { describe("Create the credential provider"); @@ -170,6 +172,7 @@ public void testCreateCredentialProviderNoURI() throws IOException { * @return a configuration set to use to the role ARN. * @throws JsonProcessingException problems working with JSON policies. */ + @SuppressWarnings("deprecation") protected Configuration createValidRoleConf() throws JsonProcessingException { String roleARN = getAssumedRoleARN(); @@ -183,6 +186,7 @@ protected Configuration createValidRoleConf() throws JsonProcessingException { } @Test + @SuppressWarnings("deprecation") public void testAssumedInvalidRole() throws Throwable { Configuration conf = new Configuration(); conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE); @@ -200,6 +204,7 @@ public void testAssumeRoleFSBadARN() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleNoARN() throws Exception { describe("Attemnpt to create the FS with no ARN"); Configuration conf = createAssumedRoleConfig(); @@ -232,6 +237,7 @@ public void testAssumeRoleFSBadPolicy2() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleCannotAuthAssumedRole() throws Exception { describe("Assert that you can't use assumed roles to auth assumed roles"); @@ -245,13 +251,13 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth() throws Exception { describe("Try to authenticate with a keypair with spaces"); Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "not valid"); conf.set(SECRET_KEY, "not secret"); expectFileSystemCreateFailure(conf, @@ -261,13 +267,13 @@ public void testAssumeRoleBadInnerAuth() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth2() throws Exception { describe("Try to authenticate with an invalid keypair"); Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "notvalid"); conf.set(SECRET_KEY, "notsecret"); expectFileSystemCreateFailure(conf, @@ -345,6 +351,7 @@ private Configuration createAssumedRoleConfig(String roleARN) { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleUndefined() throws Throwable { describe("Verify that you cannot instantiate the" + " AssumedRoleCredentialProvider without a role ARN"); @@ -356,6 +363,7 @@ public void testAssumeRoleUndefined() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAssumedIllegalDuration() throws Throwable { describe("Expect the constructor to fail if the session is to short"); Configuration conf = new Configuration(); @@ -529,6 +537,7 @@ public Path methodPath() throws IOException { * don't break. */ @Test + @SuppressWarnings("deprecation") public void testAssumedRoleRetryHandler() throws Throwable { try(AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(getFileSystem().getUri(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java index 186887d745..37c2dce4e1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java @@ -146,6 +146,7 @@ public static void assertTouchForbidden(final FileSystem fs, final Path path) * @param roleARN ARN of role * @return the new configuration */ + @SuppressWarnings("deprecation") public static Configuration newAssumedRoleConfig( final Configuration srcConf, final String roleARN) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 03b7a0b07a..9598ef084f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -254,6 +254,7 @@ public void testGetDTfromFileSystem() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAddTokensFromFileSystem() throws Throwable { describe("verify FileSystem.addDelegationTokens() collects tokens"); S3AFileSystem fs = getFileSystem(); @@ -576,6 +577,7 @@ public void testDelegationBindingMismatch2() throws Throwable { * @return result of the HEAD * @throws Exception failure */ + @SuppressWarnings("deprecation") protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { AWSCredentialProviderList testingCreds diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index 7a70ac95fc..fab7ffdbb7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -249,7 +249,7 @@ public void testCreateWithRenewer() throws Throwable { * @return the retrieved DT. This is only for error reporting. * @throws IOException failure. */ - @SuppressWarnings("OptionalGetWithoutIsPresent") + @SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"}) protected AbstractS3ATokenIdentifier verifyCredentialPropagation( final S3AFileSystem fs, final MarshalledCredentials session, @@ -259,8 +259,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( // clear any credential paths to ensure they don't get picked up and used // for authentication. unsetHadoopCredentialProviders(conf); - conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, - TemporaryAWSCredentialsProvider.NAME); + conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.NAME); session.setSecretsInConfiguration(conf); try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) { delegationTokens2.bindToFileSystem( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java new file mode 100644 index 0000000000..ffba558d11 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.test; + +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; + +import org.apache.hadoop.fs.s3a.WriteOperationHelper; + +/** + * Stub implementation of writeOperationHelper callbacks. + */ +public class MinimalWriteOperationHelperCallbacks + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return null; + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return null; + } + +}; + diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties index c831999008..0ec8d52042 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties @@ -55,6 +55,7 @@ log4j.logger.org.apache.hadoop.ipc.Server=WARN log4j.logger.org.apache.hadoop.fs.s3a=DEBUG #log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO #log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO +log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN # Log Committer classes #log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG