HADOOP-12851. S3AFileSystem Uptake of ProviderUtils.excludeIncompatibleCredentialProviders. Contributed by Larry McCay.
This commit is contained in:
parent
307ec80aca
commit
d251e55415
@ -1772,6 +1772,10 @@ Release 2.8.0 - UNRELEASED
|
||||
HADOOP-12813. Migrate TestRPC and related codes to rebase on
|
||||
ProtobufRpcEngine. (Kai Zheng via wheat9)
|
||||
|
||||
HADOOP-12851. S3AFileSystem Uptake of
|
||||
ProviderUtils.excludeIncompatibleCredentialProviders.
|
||||
(Larry McCay via cnauroth)
|
||||
|
||||
Release 2.7.3 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
@ -65,6 +65,7 @@
|
||||
import org.apache.hadoop.fs.LocalFileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.ProviderUtils;
|
||||
import org.apache.hadoop.util.Progressable;
|
||||
|
||||
import static org.apache.hadoop.fs.s3a.Constants.*;
|
||||
@ -297,9 +298,11 @@ AWSAccessKeys getAWSAccessKeys(URI name, Configuration conf)
|
||||
accessKey = userInfo;
|
||||
}
|
||||
}
|
||||
Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
|
||||
conf, S3AFileSystem.class);
|
||||
if (accessKey == null) {
|
||||
try {
|
||||
final char[] key = conf.getPassword(ACCESS_KEY);
|
||||
final char[] key = c.getPassword(ACCESS_KEY);
|
||||
if (key != null) {
|
||||
accessKey = (new String(key)).trim();
|
||||
}
|
||||
@ -309,7 +312,7 @@ AWSAccessKeys getAWSAccessKeys(URI name, Configuration conf)
|
||||
}
|
||||
if (secretKey == null) {
|
||||
try {
|
||||
final char[] pass = conf.getPassword(SECRET_KEY);
|
||||
final char[] pass = c.getPassword(SECRET_KEY);
|
||||
if (pass != null) {
|
||||
secretKey = (new String(pass)).trim();
|
||||
}
|
||||
@ -999,8 +1002,8 @@ public S3AFileStatus getFileStatus(Path f) throws IOException {
|
||||
|| objects.getObjectSummaries().size() > 0) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Found path as directory (with /): " +
|
||||
objects.getCommonPrefixes().size() + "/" +
|
||||
objects.getObjectSummaries().size());
|
||||
objects.getCommonPrefixes().size() + "/" +
|
||||
objects.getObjectSummaries().size());
|
||||
|
||||
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
|
||||
LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize());
|
||||
|
@ -29,6 +29,7 @@
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
@ -318,4 +319,37 @@ public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
|
||||
assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
|
||||
assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExcludingS3ACredentialProvider() throws Exception {
|
||||
// set up conf to have a cred provider
|
||||
final Configuration conf = new Configuration();
|
||||
final File file = tempDir.newFile("test.jks");
|
||||
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
|
||||
file.toURI());
|
||||
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
||||
"jceks://s3a/foobar," + jks.toString());
|
||||
|
||||
// first make sure that the s3a based provider is removed
|
||||
Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
|
||||
conf, S3AFileSystem.class);
|
||||
String newPath = conf.get(
|
||||
CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
|
||||
assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
|
||||
|
||||
// now let's make sure the new path is created by the S3AFileSystem
|
||||
// and the integration still works. Let's provision the keys through
|
||||
// the altered configuration instance and then try and access them
|
||||
// using the original config with the s3a provider in the path.
|
||||
provisionAccessKeys(c);
|
||||
|
||||
S3AFileSystem s3afs = new S3AFileSystem();
|
||||
conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
|
||||
URI uriWithUserInfo = new URI("s3a://123:456@foobar");
|
||||
S3AFileSystem.AWSAccessKeys creds =
|
||||
s3afs.getAWSAccessKeys(uriWithUserInfo, conf);
|
||||
assertEquals("AccessKey incorrect.", "123", creds.getAccessKey());
|
||||
assertEquals("SecretKey incorrect.", "456", creds.getAccessSecret());
|
||||
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user