From 656c460c0e79ee144d6ef48d85cec04a1af3b2cc Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 9 Jun 2016 16:36:27 +0100 Subject: [PATCH] HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth --- .../src/main/resources/core-default.xml | 13 ++++- .../s3a/AnonymousAWSCredentialsProvider.java | 11 ++++ .../fs/s3a/BasicAWSCredentialsProvider.java | 8 +++ .../apache/hadoop/fs/s3a/S3AFileSystem.java | 22 +++++--- .../site/markdown/tools/hadoop-aws/index.md | 14 ++++- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 55 +++++++++++++++++++ 6 files changed, 113 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a65246ba7d..8bb27eab32 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -791,7 +791,18 @@ fs.s3a.aws.credentials.provider - Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism. + + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index e62ec77e1b..2c863fc897 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -24,6 +24,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * AnonymousAWSCredentialsProvider supports anonymous access to AWS services + * through the AWS SDK. AWS requests will not be signed. This is not suitable + * for most cases, because allowing anonymous access to an S3 bucket compromises + * security. This can be useful for accessing public data sets without + * requiring AWS credentials. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java index 2f721e4ed8..3a5ee8c325 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java @@ -26,6 +26,14 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * BasicAWSCredentialsProvider supports static configuration of access key ID + * and secret access key for use with the AWS SDK. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class BasicAWSCredentialsProvider implements AWSCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0281a3aa37..9af0a99376 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -465,20 +465,28 @@ private AWSCredentialsProvider getAWSCredentialsProvider(URI binding, new BasicAWSCredentialsProvider( creds.getAccessKey(), creds.getAccessSecret()), new InstanceProfileCredentialsProvider(), - new EnvironmentVariableCredentialsProvider(), - new AnonymousAWSCredentialsProvider() - ); + new EnvironmentVariableCredentialsProvider()); } else { try { LOG.debug("Credential provider class is {}", className); - credentials = (AWSCredentialsProvider) Class.forName(className) - .getDeclaredConstructor(URI.class, Configuration.class) - .newInstance(this.uri, conf); + Class credClass = Class.forName(className); + try { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor( + URI.class, Configuration.class).newInstance(this.uri, conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } } catch (ClassNotFoundException e) { throw new IOException(className + " not found.", e); } catch (NoSuchMethodException | SecurityException e) { - throw new IOException(className + " constructor exception.", e); + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e); } catch (ReflectiveOperationException | IllegalArgumentException e) { throw new IOException(className + " instantiation exception.", e); } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7d63a862dd..4086bc02db 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -187,8 +187,18 @@ If you do any of these: change your credentials immediately! fs.s3a.aws.credentials.provider - Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. - Omit if using access/secret keys or another authentication mechanism. + + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + #### Protecting the AWS Credentials in S3A diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 1a11a45a19..a25ca9cd36 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; import static org.junit.Assert.*; import java.io.IOException; @@ -26,8 +27,13 @@ import java.nio.file.AccessDeniedException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; @@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider { private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); + @Rule + public Timeout testTimeout = new Timeout(1 * 60 * 1000); + + @Rule + public ExpectedException exception = ExpectedException.none(); + @Test public void testBadConfiguration() throws IOException { Configuration conf = new Configuration(); @@ -113,4 +125,47 @@ public void testGoodProvider() throws Exception { conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName()); S3ATestUtils.createTestFileSystem(conf); } + + @Test + public void testAnonymousProvider() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + AnonymousAWSCredentialsProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + assertNotNull(fs); + assertTrue(fs instanceof S3AFileSystem); + FileStatus stat = fs.getFileStatus(testFile); + assertNotNull(stat); + assertEquals(testFile, stat.getPath()); + } + + static class ConstructorErrorProvider implements AWSCredentialsProvider { + + @SuppressWarnings("unused") + public ConstructorErrorProvider(String str) { + } + + @Override + public AWSCredentials getCredentials() { + return null; + } + + @Override + public void refresh() { + } + } + + @Test + public void testProviderConstructorError() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + ConstructorErrorProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + exception.expect(IOException.class); + exception.expectMessage("constructor exception"); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + } }