big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth

Browse Source
This commit is contained in:
Steve Loughran 2016-06-09 16:36:27 +01:00
parent 8ea9bbce26
commit 656c460c0e
6 changed files with 113 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -791,7 +791,18 @@
<property>
<name>fs.s3a.aws.credentials.provider</name>
<description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism.</description>
<description>
Class name of a credentials provider that implements
com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
or another authentication mechanism. The specified class must provide an
accessible constructor accepting java.net.URI and
org.apache.hadoop.conf.Configuration, or an accessible default constructor.
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
anonymous access to a publicly accessible S3 bucket without any credentials.
Please note that allowing anonymous access to an S3 bucket compromises
security and therefore is unsuitable for most use cases. It can be useful
for accessing public data sets without requiring AWS credentials.
</description>
</property>
<property>

11
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
View File

@ -24,6 +24,17 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* AnonymousAWSCredentialsProvider supports anonymous access to AWS services
* through the AWS SDK. AWS requests will not be signed. This is not suitable
* for most cases, because allowing anonymous access to an S3 bucket compromises
* security. This can be useful for accessing public data sets without
* requiring AWS credentials.
*
* Please note that users may reference this class name from configuration
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {

8
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
View File

@ -26,6 +26,14 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* BasicAWSCredentialsProvider supports static configuration of access key ID
* and secret access key for use with the AWS SDK.
*
* Please note that users may reference this class name from configuration
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
* would be a backward-incompatible change.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
public class BasicAWSCredentialsProvider implements AWSCredentialsProvider {

22
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -465,20 +465,28 @@ private AWSCredentialsProvider getAWSCredentialsProvider(URI binding,
new BasicAWSCredentialsProvider(
creds.getAccessKey(), creds.getAccessSecret()),
new InstanceProfileCredentialsProvider(),
new EnvironmentVariableCredentialsProvider(),
new AnonymousAWSCredentialsProvider()
);
new EnvironmentVariableCredentialsProvider());
} else {
try {
LOG.debug("Credential provider class is {}", className);
credentials = (AWSCredentialsProvider) Class.forName(className)
.getDeclaredConstructor(URI.class, Configuration.class)
.newInstance(this.uri, conf);
Class<?> credClass = Class.forName(className);
try {
credentials =
(AWSCredentialsProvider)credClass.getDeclaredConstructor(
URI.class, Configuration.class).newInstance(this.uri, conf);
} catch (NoSuchMethodException | SecurityException e) {
credentials =
(AWSCredentialsProvider)credClass.getDeclaredConstructor()
.newInstance();
}
} catch (ClassNotFoundException e) {
throw new IOException(className + " not found.", e);
} catch (NoSuchMethodException | SecurityException e) {
throw new IOException(className + " constructor exception.", e);
throw new IOException(String.format("%s constructor exception. A "
+ "class specified in %s must provide an accessible constructor "
+ "accepting URI and Configuration, or an accessible default "
+ "constructor.", className, AWS_CREDENTIALS_PROVIDER), e);
} catch (ReflectiveOperationException | IllegalArgumentException e) {
throw new IOException(className + " instantiation exception.", e);
}

14
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -187,8 +187,18 @@ If you do any of these: change your credentials immediately!
<property>
<name>fs.s3a.aws.credentials.provider</name>
<description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider.
Omit if using access/secret keys or another authentication mechanism.</description>
<description>
Class name of a credentials provider that implements
com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
or another authentication mechanism. The specified class must provide an
accessible constructor accepting java.net.URI and
org.apache.hadoop.conf.Configuration, or an accessible default constructor.
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
anonymous access to a publicly accessible S3 bucket without any credentials.
Please note that allowing anonymous access to an S3 bucket compromises
security and therefore is unsuitable for most use cases. It can be useful
for accessing public data sets without requiring AWS credentials.
</description>
</property>
#### Protecting the AWS Credentials in S3A

55
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.junit.Assert.*;
import java.io.IOException;
@ -26,8 +27,13 @@
import java.nio.file.AccessDeniedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider {
private static final Logger LOG =
LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
@Rule
public Timeout testTimeout = new Timeout(1 * 60 * 1000);
@Rule
public ExpectedException exception = ExpectedException.none();
@Test
public void testBadConfiguration() throws IOException {
Configuration conf = new Configuration();
@ -113,4 +125,47 @@ public void testGoodProvider() throws Exception {
conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
S3ATestUtils.createTestFileSystem(conf);
}
@Test
public void testAnonymousProvider() throws Exception {
Configuration conf = new Configuration();
conf.set(AWS_CREDENTIALS_PROVIDER,
AnonymousAWSCredentialsProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
assertNotNull(fs);
assertTrue(fs instanceof S3AFileSystem);
FileStatus stat = fs.getFileStatus(testFile);
assertNotNull(stat);
assertEquals(testFile, stat.getPath());
}
static class ConstructorErrorProvider implements AWSCredentialsProvider {
@SuppressWarnings("unused")
public ConstructorErrorProvider(String str) {
}
@Override
public AWSCredentials getCredentials() {
return null;
}
@Override
public void refresh() {
}
}
@Test
public void testProviderConstructorError() throws Exception {
Configuration conf = new Configuration();
conf.set(AWS_CREDENTIALS_PROVIDER,
ConstructorErrorProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
exception.expect(IOException.class);
exception.expectMessage("constructor exception");
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
}
}