From b8305e6d06cbb9d44c728da14506d5cf825c12e5 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Wed, 12 Apr 2017 10:02:13 -0700 Subject: [PATCH] HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu. --- .../src/main/resources/core-default.xml | 9 +-- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +-- ...redInstanceProfileCredentialsProvider.java | 67 ------------------- .../site/markdown/tools/hadoop-aws/index.md | 33 +++------ .../fs/s3a/TestS3AAWSCredentialsProvider.java | 4 +- 5 files changed, 13 insertions(+), 108 deletions(-) delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 03e4996910..4f37c6536d 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -955,13 +955,8 @@ configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 6a11699ba5..5ff9321dd5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -339,15 +339,9 @@ public static AWSCredentialProviderList createAWSCredentialProviderSet( credentials.add(new BasicAWSCredentialsProvider( creds.getUser(), creds.getPassword())); credentials.add(new EnvironmentVariableCredentialsProvider()); - credentials.add( - SharedInstanceProfileCredentialsProvider.getInstance()); + credentials.add(InstanceProfileCredentialsProvider.getInstance()); } else { for (Class aClass : awsClasses) { - if (aClass == InstanceProfileCredentialsProvider.class) { - LOG.debug("Found {}, but will use {} instead.", aClass.getName(), - SharedInstanceProfileCredentialsProvider.class.getName()); - aClass = SharedInstanceProfileCredentialsProvider.class; - } credentials.add(createAWSCredentialProvider(conf, aClass)); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java deleted file mode 100644 index cbc07873f0..0000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import com.amazonaws.auth.InstanceProfileCredentialsProvider; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * A subclass of {@link InstanceProfileCredentialsProvider} that enforces - * instantiation of only a single instance. - * This credential provider calls the EC2 instance metadata service to obtain - * credentials. For highly multi-threaded applications, it's possible that - * multiple instances call the service simultaneously and overwhelm it with - * load. The service handles this by throttling the client with an HTTP 429 - * response or forcibly terminating the connection. Forcing use of a single - * instance reduces load on the metadata service by allowing all threads to - * share the credentials. The base class is thread-safe, and there is nothing - * that varies in the credentials across different instances of - * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton - * instance is safe. - * - * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After - * Hadoop upgrades to that version or higher, it's likely that we can remove - * this class. - */ -@InterfaceAudience.Private -@InterfaceStability.Stable -public final class SharedInstanceProfileCredentialsProvider - extends InstanceProfileCredentialsProvider { - - private static final SharedInstanceProfileCredentialsProvider INSTANCE = - new SharedInstanceProfileCredentialsProvider(); - - /** - * Returns the singleton instance. - * - * @return singleton instance - */ - public static SharedInstanceProfileCredentialsProvider getInstance() { - return INSTANCE; - } - - /** - * Default constructor, defined explicitly as private to enforce singleton. - */ - private SharedInstanceProfileCredentialsProvider() { - super(); - } -} diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 18c0cebdcb..a3a0bb1cf9 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. @@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to obtain an AWS login from a different source in the system, including environment variables, JVM properties and configuration files. -There are four AWS Credential Providers inside the `hadoop-aws` JAR: +There are three AWS Credential Providers inside the `hadoop-aws` JAR: | classname | description | |-----------|-------------| | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials | | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials | -| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service. (See below.) | | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login | There are also many in the Amazon SDKs, in particular two which are automatically @@ -425,24 +419,13 @@ set up in the authentication chain: | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables | -*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`* +*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`* Applications running in EC2 may associate an IAM role with the VM and query the [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for credentials to access S3. Within the AWS SDK, this functionality is -provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded -applications may trigger a high volume of calls to the instance metadata service -and trigger throttling: either an HTTP 429 response or a forcible close of the -connection. - -To mitigate against this problem, `hadoop-aws` ships with a variant of -`InstanceProfileCredentialsProvider` called -`SharedInstanceProfileCredentialsProvider`. Using this ensures that all -instances of S3A reuse the same instance profile credentials instead of issuing -a large volume of redundant metadata service calls. If -`fs.s3a.aws.credentials.provider` refers to -`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses -`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead. +provided by `InstanceProfileCredentialsProvider`, which internally enforces a +singleton instance in order to prevent throttling problem. *Session Credentials with `TemporaryAWSCredentialsProvider`* @@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, com.amazonaws.auth.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider @@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2. ```xml fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 33740c8ab9..82a8b841ca 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -114,7 +114,7 @@ public void testDefaultChain() throws Exception { Arrays.asList( BasicAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class); + InstanceProfileCredentialsProvider.class); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2), @@ -128,7 +128,7 @@ public void testConfiguredChain() throws Exception { List> expectedClasses = Arrays.asList( EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class, + InstanceProfileCredentialsProvider.class, AnonymousAWSCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(