big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)

Browse Source
This commit is contained in:
Steve Loughran 2015-03-08 11:20:42 -07:00
parent 608ebd52ba
commit 64443490d7
4 changed files with 26 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1050,6 +1050,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11674. oneByteBuf in CryptoInputStream and CryptoOutputStream HADOOP-11674. oneByteBuf in CryptoInputStream and CryptoOutputStream
should be non static. (Sean Busbey via yliu) should be non static. (Sean Busbey via yliu)
HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
Release 2.6.1 - UNRELEASED Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -18,8 +18,12 @@
package org.apache.hadoop.fs.s3a; package org.apache.hadoop.fs.s3a;
public class Constants { public class Constants {
// s3 access key
public static final String ACCESS_KEY = "fs.s3a.access.key";
// s3 secret key
public static final String SECRET_KEY = "fs.s3a.secret.key";
// number of simultaneous connections to s3 // number of simultaneous connections to s3
public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum"; public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";

20
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -32,8 +32,6 @@
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.s3.S3Credentials;
import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException; import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration; import com.amazonaws.ClientConfiguration;
@ -159,12 +157,22 @@ public void initialize(URI name, Configuration conf) throws IOException {
this.getWorkingDirectory()); this.getWorkingDirectory());
// Try to get our credentials or just connect anonymously // Try to get our credentials or just connect anonymously
S3Credentials s3Credentials = new S3Credentials(); String accessKey = conf.get(ACCESS_KEY, null);
s3Credentials.initialize(name, conf); String secretKey = conf.get(SECRET_KEY, null);
String userInfo = name.getUserInfo();
if (userInfo != null) {
int index = userInfo.indexOf(':');
if (index != -1) {
accessKey = userInfo.substring(0, index);
secretKey = userInfo.substring(index + 1);
} else {
accessKey = userInfo;
}
}
AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
new BasicAWSCredentialsProvider(s3Credentials.getAccessKey(), new BasicAWSCredentialsProvider(accessKey, secretKey),
s3Credentials.getSecretAccessKey()),
new InstanceProfileCredentialsProvider(), new InstanceProfileCredentialsProvider(),
new AnonymousAWSCredentialsProvider() new AnonymousAWSCredentialsProvider()
); );

10
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -141,12 +141,12 @@ If you do any of these: change your credentials immediately!
### Authentication properties ### Authentication properties
<property> <property>
<name>fs.s3a.awsAccessKeyId</name> <name>fs.s3a.access.key</name>
<description>AWS access key ID. Omit for Role-based authentication.</description> <description>AWS access key ID. Omit for Role-based authentication.</description>
</property> </property>
<property> <property>
<name>fs.s3a.awsSecretAccessKey</name> <name>fs.s3a.secret.key</name>
<description>AWS secret key. Omit for Role-based authentication.</description> <description>AWS secret key. Omit for Role-based authentication.</description>
</property> </property>
@ -411,13 +411,13 @@ Example:
</property> </property>
<property> <property>
<name>fs.s3a.awsAccessKeyId</name> <name>fs.s3a.access.key</name>
<description>AWS access key ID. Omit for Role-based authentication.</description> <description>AWS access key ID. Omit for Role-based authentication.</description>
<value>DONOTPCOMMITTHISKEYTOSCM</value> <value>DONOTCOMMITTHISKEYTOSCM</value>
</property> </property>
<property> <property>
<name>fs.s3a.awsSecretAccessKey</name> <name>fs.s3a.secret.key</name>
<description>AWS secret key. Omit for Role-based authentication.</description> <description>AWS secret key. Omit for Role-based authentication.</description>
<value>DONOTEVERSHARETHISSECRETKEY!</value> <value>DONOTEVERSHARETHISSECRETKEY!</value>
</property> </property>